| @@ -1,844 +1,169 @@ |
| package datadownload; |
| |
| -import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; |
| -import com.gargoylesoftware.htmlunit.WebClient; |
| -import com.gargoylesoftware.htmlunit.WebWindow; |
| -import com.gargoylesoftware.htmlunit.WebWindowEvent; |
| -import com.gargoylesoftware.htmlunit.WebWindowListener; |
| -import com.gargoylesoftware.htmlunit.html.HtmlButtonInput; |
| -import com.gargoylesoftware.htmlunit.html.HtmlForm; |
| -import com.gargoylesoftware.htmlunit.html.HtmlInput; |
| -import com.gargoylesoftware.htmlunit.html.HtmlOption; |
| -import com.gargoylesoftware.htmlunit.html.HtmlPage; |
| -import com.gargoylesoftware.htmlunit.html.HtmlRadioButtonInput; |
| -import com.gargoylesoftware.htmlunit.html.HtmlSelect; |
| import java.io.BufferedReader; |
| import java.io.File; |
| -import java.io.FileNotFoundException; |
| -import java.io.FileOutputStream; |
| import java.io.FileReader; |
| import java.io.IOException; |
| -import java.io.InputStream; |
| -import java.net.MalformedURLException; |
| -import java.net.URL; |
| +import java.io.InputStreamReader; |
| import java.util.ArrayList; |
| -import java.util.Enumeration; |
| -import java.util.Iterator; |
| -import java.util.LinkedList; |
| import java.util.List; |
| -import java.util.zip.ZipEntry; |
| -import java.util.zip.ZipFile; |
| |
| /** |
| -* Last Updated: 16-December-2014 |
| +* Last Updated: 16-February-2016 |
| * @author Tyler Wible |
| * @since 22-June-2012 |
| */ |
| public class STORET_Data { |
| - final LinkedList<WebWindow> windows = new LinkedList<WebWindow>(); |
| - /** |
| - * Sub-function to convert month number into a month text |
| - * @param month_String the month number (Ex. 01). |
| - * @return month text (Ex. JAN). |
| - */ |
| - public String ConvertMonth (String month_String){ |
| - String month_number = "JAN"; |
| - if (month_String.equalsIgnoreCase("01")){ |
| - month_number = "JAN"; |
| - }else if (month_String.equalsIgnoreCase("02")){ |
| - month_number = "FEB"; |
| - }else if (month_String.equalsIgnoreCase("03")){ |
| - month_number = "MAR"; |
| - }else if (month_String.equalsIgnoreCase("04")){ |
| - month_number = "APR"; |
| - }else if (month_String.equalsIgnoreCase("05")){ |
| - month_number = "MAY"; |
| - }else if (month_String.equalsIgnoreCase("06")){ |
| - month_number = "JUN"; |
| - }else if (month_String.equalsIgnoreCase("07")){ |
| - month_number = "JUL"; |
| - }else if (month_String.equalsIgnoreCase("08")){ |
| - month_number = "AUG"; |
| - }else if (month_String.equalsIgnoreCase("09")){ |
| - month_number = "SEP"; |
| - }else if (month_String.equalsIgnoreCase("10")){ |
| - month_number = "OCT"; |
| - }else if (month_String.equalsIgnoreCase("11")){ |
| - month_number = "NOV"; |
| - }else if (month_String.equalsIgnoreCase("12")){ |
| - month_number = "DEC"; |
| - } |
| - return month_number; |
| - } |
| - /** |
| - * Sub-function to convert month number into a month text |
| - * @param month_String the month number (Ex. 01). |
| - * @return month text (Ex. JAN). |
| - */ |
| - public String ConvertDay (String day_String){ |
| - if (day_String.equalsIgnoreCase("01")){ |
| - day_String = "1"; |
| - }else if (day_String.equalsIgnoreCase("02")){ |
| - day_String = "2"; |
| - }else if (day_String.equalsIgnoreCase("03")){ |
| - day_String = "3"; |
| - }else if (day_String.equalsIgnoreCase("04")){ |
| - day_String = "4"; |
| - }else if (day_String.equalsIgnoreCase("05")){ |
| - day_String = "5"; |
| - }else if (day_String.equalsIgnoreCase("06")){ |
| - day_String = "6"; |
| - }else if (day_String.equalsIgnoreCase("07")){ |
| - day_String = "7"; |
| - }else if (day_String.equalsIgnoreCase("08")){ |
| - day_String = "8"; |
| - }else if (day_String.equalsIgnoreCase("09")){ |
| - day_String = "9"; |
| - } |
| - return day_String; |
| - } |
| - /** |
| - * Get list of information contained in listbox on the specified page with the specified attribute "name" |
| - * @param select The HtmlSelect listbox |
| - * @param listboxType Which listbox is being interacted with, |
| - * the organization names, station names search |
| - * @param name The current user selection which is being searched for in the listbox |
| - */ |
| - public void selectListboxData(HtmlSelect select, String listboxType, String name){ |
| - //Iterate through the options of the select box, find and select the organization specified |
| - List<HtmlOption> selectOptions = select.getOptions(); |
| - Iterator<HtmlOption> listIterate = selectOptions.iterator(); |
| - int ctr = 0; |
| - while(listIterate.hasNext()){ |
| - HtmlOption currentOption = listIterate.next(); |
| - String optionText1 = currentOption.getText(); |
| - System.out.println(optionText1); |
| - if(ctr > 1){//Skip the first two lines |
| - String optionText = currentOption.getText(); |
| - |
| - String optionTextClip = optionText; |
| - if(!listboxType.equals("Characteristics")){//Only trim organizations and stations |
| - int index1 = optionText.indexOf(" "); |
| - if(index1 != -1){ |
| - optionTextClip = optionText.substring(index1); |
| - optionTextClip = optionTextClip.trim(); |
| - } |
| - } |
| - System.out.println(optionTextClip); |
| - |
| - //If the current option is the desired one, select it |
| - if(optionTextClip.toLowerCase().contains(name.toLowerCase())){ |
| - //If the current select option is the organization name, then select it and break the loop |
| - String org_index = currentOption.getValueAttribute(); |
| - select.setSelectedAttribute(org_index, true); |
| - System.out.println(listboxType + " selected"); |
| - break; |
| - } |
| - } |
| - ctr++; |
| - } |
| - } |
| - /** |
| - * Assembles an error message to be returned early if an error occurs and the page isn't working properly. |
| - * @param error_number The error code to be displayed |
| - * @return a string containing the error message and number |
| - */ |
| - public String ExitSystem(String error_number){ |
| - String message = "An unexpected error occured while downloading your requested data. Report '" + error_number + "' to the webmaster"; |
| - return message; |
| - } |
| - /** |
| - * Writes out the error message, if any, for finding the file and then exits the program |
| - * @param error string array to be written as each line of an error message |
| - * @throws IOException |
| - */ |
| - public void writeError(String[] error) throws IOException{ |
| - //Output data to text file |
| - String errorContents = error[0]; |
| - for(int i=1; i<error.length; i++){ |
| - errorContents = errorContents + "\n" + error[i]; |
| - } |
| - throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents); |
| - } |
| - /** |
| - * Tries to access the url zip download and downloads the zip once it exists |
| - * @param downloadUrl The URL for the ZIP data download page |
| - * @param downloadfile The name of the downloaded ZIP file |
| - * @param mainFolder the folder in which to download the file |
| - * @return the zipfile location |
| - */ |
| - public String downloadzip(String downloadUrl, String downloadfile, String mainFolder){ |
| - System.out.println("File name: " + downloadfile); |
| - System.out.println("Website: " + downloadUrl); |
| - String zipfile_location = mainFolder + "/" + downloadfile; |
| - try{//Get a connection to the URL |
| - boolean exists = false; |
| - URL url = new URL(downloadUrl); |
| - url.openConnection(); |
| - InputStream reader = null; |
| - //Loop until the website URL exists |
| - int ctr=0; |
| - while(!exists){ |
| - try{ |
| - reader = url.openStream(); |
| - System.out.println("Step 9ba: website download exists"); |
| - exists = true; |
| - }catch(FileNotFoundException e){ |
| - try{ |
| - System.out.println("Try: " + ctr + " download website does not exist"); |
| - ctr++; |
| - Thread.sleep(5000); // do nothing for 5000 milliseconds (5 second) |
| - |
| - //Return an error if the webpage didn't respond given enough time |
| - if(ctr > 180){ |
| - String message = "Error STORET FLDC0014. Waiting for the webpage (" + downloadUrl + ") timed out please try again"; |
| - return message; |
| - } |
| - }catch(InterruptedException f){ |
| - f.printStackTrace(); |
| - } |
| - } |
| - } |
| - //Start a buffered file writer to output the result of the download file |
| - FileOutputStream writer = new FileOutputStream(zipfile_location); |
| - byte[] buffer = new byte[153600]; |
| - int bytesRead = 0; |
| - //read the file 150KB at a time |
| - while ((bytesRead = reader.read(buffer)) > 0){ |
| - writer.write(buffer, 0, bytesRead); |
| - buffer = new byte[153600]; |
| - } |
| - //Close the reader and writer |
| - writer.close(); |
| - reader.close(); |
| - }catch(MalformedURLException e){ |
| - e.printStackTrace(); |
| - }catch (IOException e){ |
| - e.printStackTrace(); |
| - } |
| - System.out.println("Step 9c: finished downloading file"); |
| - return zipfile_location; |
| - } |
| /** |
| * Interfaces with the STORET download website and searches for the data for a specific station and |
| * read it out into a variable depending on what data was requested |
| * @param mainFolder file location to download the resulting STORET zip file into |
| - * @param organizationName name of the supervizing agency for the current station, used to search the STORET database |
| + * @param resourceFile the csip resource file for the python driver for the storet rest call |
| + * @param cfaFormat_FT if true the internal csip-cfa data format is provided (date\twq_test\r\n1999-01-01\t34.2...), otherwise the STORET data download result file format is provided |
| + * @param organizationID The ID for the supervising organization for the station within STORET |
| * @param stationID station ID of the current station, used to search the STORET databes |
| - * @param characteristicLong the full name of the characteristic being searched for |
| + * @param wq_test The word 'flow' or the formatted name for the water quality test being used, Format is: '00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L' = '5-digit-USGS-water-quality-test-code test-name -- units' |
| * @param beginDate the used defined begin date of search |
| * @param endDate the user defined end date of search |
| - * @return a String containing the full path to the downloaded zip file (ex. "C:/tempFolder/myZipfile.zip") |
| + * @return a result File of the STORET rest service result formatted according to cfaFormat_TF |
| * @throws IOException |
| */ |
| - public String downloadSTORET(String mainFolder, |
| - String organizationName, |
| - String stationID, |
| - String characteristicLong, |
| - String beginDate, |
| - String endDate) throws IOException{ |
| + public File downloadSTORET(String mainFolder, |
| + File resourceFile, |
| + boolean cfaFormat_FT, |
| + String organizationID, |
| + String stationID, |
| + String wq_test, |
| + String beginDate, |
| + String endDate) throws IOException, InterruptedException{ |
| + //Setup directory/result file |
| + String file_name = "STORET_" + organizationID + "_" + stationID + "_" + wq_test + "results.txt"; |
| + file_name = file_name.replace("/", ""); |
| + File storetResultFile = new File(mainFolder, file_name); |
| + |
| + //Check dates |
| + if(!beginDate.equalsIgnoreCase("") && beginDate.compareToIgnoreCase("1900-01-01") < 0){ |
| + //Artificial limit due to the properties of the STORET website |
| + beginDate = "1900-01-01"; |
| + }else if(beginDate.equalsIgnoreCase("")){ |
| + //Because python can't parse blank values as inputs very well |
| + beginDate = "-1"; |
| + } |
| + |
| + if(!endDate.equalsIgnoreCase("") && endDate.compareToIgnoreCase("1900-01-01") < 0){ |
| + //Artificial limit due to the properties of the STORET website |
| + endDate = "1900-01-01"; |
| + }else if(endDate.equalsIgnoreCase("")){ |
| + //Because python can't parse blank values as inputs very well |
| + endDate = "-1"; |
| + } |
| + |
| + if(wq_test.equalsIgnoreCase("")){ |
| + //Because python can't parse blank values as inputs very well |
| + wq_test = "-1"; |
| + } |
| + |
| + //Setup inputs to python |
| + List<String> args = new ArrayList<String>(); |
| + args.add("python"); |
| + args.add(resourceFile.getAbsolutePath()); |
| + args.add(String.valueOf(cfaFormat_FT)); |
| + args.add(organizationID); |
| + args.add(stationID); |
| + args.add(wq_test); |
| + args.add(beginDate); |
| + args.add(endDate); |
| + args.add(storetResultFile.getAbsolutePath()); |
| |
| - String characteristicShort = characteristicLong; |
| + //Call service |
| + ProcessBuilder pb = new ProcessBuilder(args); |
| + Process p = pb.start(); |
| + BufferedReader bfr_error = new BufferedReader(new InputStreamReader(p.getErrorStream())); |
| + int exitCode = p.waitFor(); |
| |
| - //Artificial limit due to the properties of the STORET website |
| - if(beginDate.compareToIgnoreCase("1900-01-01") < 0){ |
| - beginDate = "1900-01-01"; |
| + String line; |
| + String error_log = ""; |
| + while ((line = bfr_error.readLine()) != null) { |
| + error_log += line + "\n"; |
| + System.err.println(error_log); |
| } |
| - if(endDate.compareToIgnoreCase("1900-01-01") < 0){ |
| - endDate = "1900-01-01"; |
| - } |
| - |
| - //Edit inputs to fit search criteria and to get extra info |
| - String email_address = "eRAMSInfo@gmail.com"; |
| - String start_year = beginDate.substring(0,4); |
| - String start_month = beginDate.substring(5,7); |
| - start_month = ConvertMonth(start_month); |
| - String start_day = beginDate.substring(8); |
| - start_day = ConvertDay(start_day); |
| - String end_year = endDate.substring(0,4); |
| - String end_month = endDate.substring(5,7); |
| - end_month = ConvertMonth(end_month); |
| - String end_day = endDate.substring(8); |
| - end_day = ConvertDay(end_day); |
| - if(characteristicLong.length() > 20){ |
| - characteristicShort = characteristicLong.substring(0,20); |
| - } |
| - //old |
| - //String mainUrl = "http://iaspub.epa.gov/storpubl/DW_resultcriteria_station"; |
| - //new |
| - String mainUrl = "http://ofmpub.epa.gov/storpubl/dw_pages.resultcriteria"; |
| - |
| - |
| - //Create Webclient with specific properties for STORET webpage |
| - WebClient webClient = new WebClient(); |
| - webClient.setThrowExceptionOnScriptError(false); |
| - webClient.addWebWindowListener(new WebWindowListener(){ |
| - public void webWindowClosed(WebWindowEvent event){ |
| - } |
| - public void webWindowContentChanged(WebWindowEvent event){ |
| - } |
| - public void webWindowOpened(WebWindowEvent event){ |
| - windows.add(event.getWebWindow()); |
| - } |
| - }); |
| - |
| - |
| - //Get STORET "Results by Station" webpage |
| - HtmlPage mainPage = null; |
| - try { |
| - mainPage = webClient.getPage(mainUrl); |
| - }catch (FailingHttpStatusCodeException e) { |
| - e.printStackTrace(); |
| - }catch (MalformedURLException e) { |
| - e.printStackTrace(); |
| - }catch (IOException e) { |
| - e.printStackTrace(); |
| - } |
| - if(mainPage == null){ |
| - return ExitSystem("Error Data_STORET_0001"); |
| - } |
| - System.out.println("Step 1: on webpage"); |
| - |
| - //Selecct Search Type |
| - List<?> buttonList = (List<?>) mainPage.getByXPath("//input[@value='Organization and Station']"); |
| - if(buttonList.size() == 1){ |
| - System.out.println(buttonList.size()); |
| - HtmlRadioButtonInput searchTypeRadioButton = (HtmlRadioButtonInput) buttonList.get(0); |
| - searchTypeRadioButton.focus(); |
| - mainPage = searchTypeRadioButton.click();//onclick javascript function will check this radio button and uncheck the others |
| -// searchTypeRadioButton.setChecked(true); |
| - |
| - System.out.println(searchTypeRadioButton.getValueAttribute()); |
| - System.out.println(searchTypeRadioButton.getCheckedAttribute()); |
| - System.out.println("Step 2: selected searchtype"); |
| - } |
| - |
| - //Select Organization ID |
| - List<?> formList = (List<?>) mainPage.getForms(); |
| - if(formList.size() >= 2){ |
| - HtmlForm searchForm = (HtmlForm) formList.get(1); |
| - HtmlSelect select = (HtmlSelect)searchForm.getSelectsByName("D_ORG_LIST").get(0); |
| - select.focus(); |
| - selectListboxData(select,"Organizations",organizationName); |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0002"); |
| - } |
| - System.out.println("Step 3: Try Selecting a station"); |
| - |
| - //Input Station ID |
| - HtmlInput queryInput2 = mainPage.getElementByName("as_station"); |
| - queryInput2.setValueAttribute(stationID); |
| - |
| - //Search Organization for current Station ID (push "Search Stations" button and get pop-up window) |
| - List<?> popupList = (List<?>) mainPage.getByXPath("//input[@value='Search Stations']"); |
| - if (popupList.size() == 1){ |
| - try{ |
| - HtmlButtonInput popup = (HtmlButtonInput) popupList.get(0); |
| - popup.focus(); |
| - HtmlPage popupPage_sta = popup.click(); |
| - |
| - //Select Station ID |
| - List<?> formList2 = (List<?>) popupPage_sta.getForms(); |
| - HtmlForm searchForm2 = (HtmlForm) formList2.get(0); |
| - HtmlSelect select2 = (HtmlSelect)searchForm2.getSelectsByName("d_station_list").get(0); |
| - selectListboxData(select2,"Stations",stationID); |
| - |
| - //Get update main page by pushing "Select" button on pop-up |
| - List<?> list = (List<?>) popupPage_sta.getByXPath("//input[@value='Select']"); |
| - if (list.size() == 1){ |
| - try{ |
| - HtmlButtonInput select_button = (HtmlButtonInput) list.get(0); |
| - select_button.focus(); |
| - mainPage = select_button.click(); |
| - }catch(Exception error){ |
| - System.err.println(error.toString()); |
| - } |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0004"); |
| - } |
| - }catch(Exception error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0005"); |
| - } |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0003"); |
| - } |
| - System.out.println("Step 4: selected station from popup"); |
| - |
| - //Select Date Range |
| - |
| - //Select Start year |
| - HtmlForm searchForm3 = (HtmlForm) formList.get(1); |
| - HtmlSelect select3 = (HtmlSelect)searchForm3.getSelectsByName("FROM_YYYY1").get(0); |
| - mainPage = (HtmlPage) select3.setSelectedAttribute(start_year, true); |
| - |
| - //Select Start month |
| - HtmlSelect select4 = (HtmlSelect)searchForm3.getSelectsByName("FROM_MON1").get(0); |
| - mainPage = (HtmlPage) select4.setSelectedAttribute(start_month, true); |
| - |
| - //Select Start day |
| - HtmlSelect select5 = (HtmlSelect)searchForm3.getSelectsByName("FROM_DD1").get(0); |
| - mainPage = (HtmlPage) select5.setSelectedAttribute(start_day, true); |
| - |
| - //Select End year |
| - HtmlSelect select6 = (HtmlSelect)searchForm3.getSelectsByName("END_YYYY1").get(0); |
| - mainPage = (HtmlPage) select6.setSelectedAttribute(end_year, true); |
| - |
| - //Select End month |
| - HtmlSelect select7 = (HtmlSelect)searchForm3.getSelectsByName("END_MON1").get(0); |
| - mainPage = (HtmlPage) select7.setSelectedAttribute(end_month, true); |
| - |
| - //Select End day |
| - HtmlSelect select8 = (HtmlSelect)searchForm3.getSelectsByName("END_DD1").get(0); |
| - mainPage = (HtmlPage) select8.setSelectedAttribute(end_day, true); |
| - |
| - System.out.println("Step 5: selected dates"); |
| - |
| - //Input Characteristic |
| - if(!characteristicShort.equalsIgnoreCase("flow") && !characteristicShort.equalsIgnoreCase("all")){ |
| - //If the search characteristic is not flow or all then search for the current characteristic |
| - HtmlInput queryInput9 = mainPage.getElementByName("as_char"); |
| - queryInput9.setValueAttribute(characteristicShort); |
| - |
| - //Search Characteristic for current characteristic (push "Search" button and get pop-up window) |
| - List<?> popupList9 = (List<?>) mainPage.getByXPath("//input[@value='Search']"); |
| - try{ |
| - //Select first value in characteristic "search by" list box which should be "Characteristic Name" |
| - HtmlForm searchForm9 = (HtmlForm) formList.get(1); |
| - HtmlSelect select9 = (HtmlSelect)searchForm9.getSelectsByName("as_char_alias_type").get(0); |
| - select9.setSelectedAttribute("0", true); |
| - |
| - //Push "Search" button and get pop-up window |
| - HtmlButtonInput popup_cha= (HtmlButtonInput) popupList9.get(1); |
| - popup_cha.focus(); |
| - HtmlPage popupPage_cha = popup_cha.click(); |
| - |
| - //Select Characteristic |
| - List<?> formList10 = (List<?>) popupPage_cha.getForms(); |
| - HtmlForm searchForm10 = (HtmlForm) formList10.get(0); |
| - HtmlSelect select10 = (HtmlSelect)searchForm10.getSelectsByName("as_char").get(0); |
| - selectListboxData(select10,"Characteristics",characteristicLong); |
| - |
| - //Get update main page by pushing "Select" button on pop-up |
| - List<?> list = (List<?>) popupPage_cha.getByXPath("//input[@value='Select']"); |
| - if (list.size() == 1){ |
| - try{ |
| - HtmlButtonInput select_button = (HtmlButtonInput) list.get(0); |
| - select_button.focus(); |
| - mainPage = select_button.click(); |
| - }catch(Exception error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0008"); |
| - } |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0007"); |
| - } |
| - }catch(Exception error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0006"); |
| - } |
| - System.out.println("Step 6a: selected a characteristic"); |
| - } |
| - |
| - |
| - |
| - //Input flow characteristic |
| - if(!characteristicShort.equalsIgnoreCase("all")){ |
| - //If the search characteristic is not all then search for flow data, |
| - // not performing this operation defaults to all data for a STORET station, |
| - //which is used by the "data download" version of this model |
| - HtmlInput queryInput11 = mainPage.getElementByName("as_char"); |
| - queryInput11.setValueAttribute("Flow"); |
| - |
| - //Search Characteristic for current characteristic (push "Search" button and get pop-up window) |
| - List<?> popupList11 = (List<?>) mainPage.getByXPath("//input[@value='Search']"); |
| - try{ |
| - //Select first value in characteristic "search by" list box which should be "Characteristic Name" |
| - HtmlForm searchForm11 = (HtmlForm) formList.get(1); |
| - HtmlSelect select11 = (HtmlSelect)searchForm11.getSelectsByName("as_char_alias_type").get(0); |
| - select11.setSelectedAttribute("0", true); |
| - |
| - //Push "Search" button and get pop-up window |
| - HtmlButtonInput popup_cha= (HtmlButtonInput) popupList11.get(1); |
| - popup_cha.focus(); |
| - HtmlPage popupPage_cha = popup_cha.click(); |
| - |
| - //Select Characteristic |
| - List<?> formList12 = (List<?>) popupPage_cha.getForms(); |
| - HtmlForm searchForm12 = (HtmlForm) formList12.get(0); |
| - HtmlSelect select12 = (HtmlSelect)searchForm12.getSelectsByName("as_char").get(0); |
| - selectListboxData(select12,"Flow Characteristic","Flow"); |
| - |
| - //Get update main page by pushing "Select" button on pop-up |
| - List<?> list = (List<?>) popupPage_cha.getByXPath("//input[@value='Select']"); |
| - if (list.size() == 1){ |
| - try{ |
| - HtmlButtonInput select_button = (HtmlButtonInput) list.get(0); |
| - select_button.focus(); |
| - mainPage = select_button.click(); |
| - }catch(Exception error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0008"); |
| - } |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0007"); |
| - } |
| - }catch(Exception error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0006"); |
| - } |
| - System.out.println("Step 6b: selected flow characteristic"); |
| - } |
| - |
| - |
| - |
| -//FileWriter write = new FileWriter(mainFolder + "/page1.txt", false); |
| -//PrintWriter print_line = new PrintWriter(write); |
| -//print_line.printf("%s" + "\r\n", mainPage.asXml()); |
| -//print_line.close(); |
| - |
| - //Select "Continue" Button |
| - List<?> buttonList2 = (List<?>) mainPage.getByXPath("//input[@value='Continue']"); |
| - HtmlPage resultPage = null; |
| - try{ |
| - if(buttonList2.size() == 1){ |
| - HtmlButtonInput select11 = (HtmlButtonInput) buttonList2.get(0); |
| - select11.focus(); |
| - resultPage = select11.click(); |
| - }else{ |
| - return ExitSystem("Error Data_STORET_0009"); |
| - } |
| - }catch(IOException error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0010"); |
| - } |
| - System.out.println("Step 7: submitted request, now entering final inputs"); |
| - String xmlSource_result = ""; |
| - xmlSource_result = resultPage.asXml(); |
| - |
| - |
| - //Exit if there are no results or too many to download "Immediately" |
| - if(xmlSource_result.contains("No Results match") || xmlSource_result.contains("The number of Results that match your search criteria has exceeded")){ |
| - String message = "Error: There is no available data for station '" + stationID + "' and the specified date range"; |
| - return message; |
| - } |
| - |
| - |
| - //Enter email address |
| -// HtmlSelect queryInput12 = resultPage.getElementByName("v_userprofile"); |
| -// queryInput12.setSelectedAttribute("8", true);//Select value=8 which is the public user |
| - HtmlInput queryInput13 = resultPage.getElementByName("v_email"); |
| - queryInput13.setValueAttribute(email_address); |
| - HtmlInput queryInput14 = resultPage.getElementByName("v_prefix"); |
| - queryInput14.setValueAttribute("tmd"); |
| - |
| -//write = new FileWriter(mainFolder + "/page2.txt", false); |
| -//print_line = new PrintWriter(write); |
| -//print_line.printf("%s" + "\r\n", resultPage.asXml()); |
| -//print_line.close(); |
| - |
| - |
| - //Select "Immediate" Batch Processing button |
| - List<?> buttonList3 = (List<?>) resultPage.getByXPath("//input[@value='Immediate']"); |
| - HtmlPage BatchPage = null; |
| - try{ |
| - if(buttonList3.size() == 3){ |
| - HtmlButtonInput select14 = (HtmlButtonInput) buttonList3.get(1); |
| - select14.focus(); |
| - BatchPage = select14.click(); |
| - }else{ |
| - return ExitSystem("Error v0011"); |
| - } |
| - }catch(IOException error){ |
| - error.printStackTrace(); |
| - return ExitSystem("Error Data_STORET_0012"); |
| - } |
| - System.out.println("Step 8: submitted data extraction, waiting on webpage to return"); |
| -//write = new FileWriter(mainFolder + "/page3.txt", false); |
| -//print_line = new PrintWriter(write); |
| -//print_line.printf("%s" + "\r\n", BatchPage.asXml()); |
| -//print_line.close(); |
| - |
| - //Find url to download webpage |
| - String xmlSource_batch = BatchPage.asXml(); |
| - int urlInt_begin = xmlSource_batch.indexOf("http://www.epa.gov/storpubl/modern/downloads/tmd"); |
| - int urlInt_end = xmlSource_batch.indexOf("zip", urlInt_begin); |
| - if(urlInt_begin == -1 || urlInt_end == -1){ |
| - return ExitSystem("Error Data_STORET_0013"); |
| - } |
| - String downloadUrl = ""; |
| - downloadUrl = xmlSource_batch.substring(urlInt_begin,urlInt_end + 3); |
| - String zipfile = downloadUrl.substring(downloadUrl.indexOf("tmd")); |
| - |
| - //Download STORET data |
| - System.out.println("Step 9: Determining zip-file to download"); |
| - String zipfile_location = downloadzip(downloadUrl, zipfile, mainFolder); |
| - |
| - //Close web interaction |
| - webClient.closeAllWindows(); |
| - |
| - //Check if there is an error downloading the file |
| - if(zipfile_location.contains("Error")){ |
| - String[] errorMessage = {"There is a problem extracting the data from STORET for station '" + stationID + "' by: " + |
| - organizationName, zipfile_location}; |
| - writeError(errorMessage); |
| - } |
| - |
| - //Return filename/location |
| - System.out.println("Step 10: request complete"); |
| - return zipfile_location; |
| + String temp = (error_log.isEmpty()) ? null : error_log; |
| + |
| + return storetResultFile; |
| } |
| + |
| /** |
| - * Unzips the downloaded zip-file |
| - * @param zip_location The location on the drive where the downloaded STORET zip file was saved |
| + * Interfaces with the STORET download website and searches for the data for a specific station and |
| + * read it out into a variable depending on what data was requested |
| + * @param mainFolder file location to download the resulting STORET zip file into |
| + * @param resourceFile the csip resource file for the python driver for the storet rest call |
| + * @param cfaFormat_FT if true the internal csip-cfa data format is provided (date\twq_test\r\n1999-01-01\t34.2...), otherwise the STORET data download result file format is provided |
| + * @param organizationID The ID for the supervising organization for the station within STORET |
| + * @param stationID station ID of the current station, used to search the STORET databes |
| + * @param wq_test The word 'flow' or the formatted name for the water quality test being used, Format is: '00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L' = '5-digit-USGS-water-quality-test-code test-name -- units' |
| + * @param beginDate the used defined begin date of search |
| + * @param endDate the user defined end date of search |
| + * @throws IOException |
| */ |
| - public void UnZip_file(String zip_location){ |
| - try { |
| - File mainDir = new File(zip_location.substring(0,zip_location.indexOf("tmd"))); |
| - ZipFile zipFile = new ZipFile(zip_location); |
| - Enumeration<?> enumeration = zipFile.entries(); |
| - File new_file = null; |
| - FileOutputStream fout_strm = null; |
| - //While there are multiple files inside the zip file, keep unzipping until all have been unzipped. |
| - while (enumeration.hasMoreElements()) { |
| - ZipEntry zipEntry = (ZipEntry) enumeration.nextElement(); |
| - InputStream in_strm = zipFile.getInputStream(zipEntry); |
| - byte[] buffer_bytes = new byte[1024]; |
| - int bytesRead = 0; |
| - new_file = new File(mainDir.getAbsolutePath() + File.separator + zipEntry.getName()); |
| - |
| - if(zipEntry.isDirectory()){ |
| - new_file.mkdirs(); |
| - continue; |
| - }else{ |
| - new_file.getParentFile().mkdirs(); |
| - new_file.createNewFile(); |
| - } |
| - fout_strm = new FileOutputStream(new_file); |
| - while((bytesRead = in_strm.read(buffer_bytes)) != -1){ |
| - fout_strm.write(buffer_bytes, 0, bytesRead); |
| - } |
| - if(fout_strm != null){ |
| - fout_strm.close(); |
| - } |
| - in_strm.close(); |
| - } |
| - zipFile.close(); |
| - }catch(IOException e){ |
| - e.printStackTrace(); |
| - } |
| - } |
| - /** |
| - * Opens file and pulls out water quality data for LDC |
| - * @param partialPath the folder path to the file (ex. "C:/temp/myFolder/") |
| - * @param fileName the name of the file (ex "myFile.txt") |
| - * @param Characteristic The characteristic currently searched for in the text file, either flow or the user-defined characteristic |
| - * @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue |
| - */ |
| - public String[][] OpenWQFile(String partialPath, String fileName, String Characteristic) throws IOException { |
| - //Open file |
| - FileReader fr = new FileReader(partialPath + fileName); |
| + public String[][] getSTORETdata(String mainFolder, |
| + File resourceFile, |
| + boolean cfaFormat_FT, |
| + String organizationID, |
| + String stationID, |
| + String wq_test, |
| + String beginDate, |
| + String endDate) throws IOException, InterruptedException{ |
| + //Get storet data |
| + File storetResultFile = downloadSTORET(mainFolder, resourceFile, cfaFormat_FT, organizationID, stationID, wq_test, beginDate, endDate); |
| + |
| + //Read Result file |
| + FileReader fr = new FileReader(storetResultFile); |
| BufferedReader textReader = new BufferedReader(fr); |
| |
| - ArrayList<String> textData = new ArrayList<String>(); |
| - String currentLine = ""; |
| - int ctr = 0; |
| + String storetData = ""; |
| + String currentLine; |
| while((currentLine = textReader.readLine()) !=null){ |
| - String f[] = currentLine.split("\t"); |
| - |
| - //Depending on current characteristic |
| - if((f.length >= 25) && (ctr != 0)){ |
| - //Ignore effluent data |
| - if(f[1].equalsIgnoreCase("EFFLUENT")){ |
| - continue; |
| - } |
| - //Ignore Non-detect data |
| - if(f[23].equalsIgnoreCase("*Non-detect")){ |
| - continue; |
| - } |
| - if(f[24].indexOf(" ") != -1){ |
| - f[24]= f[24].substring(0,f[24].indexOf(" ")); |
| - } |
| - if(Characteristic.equals("Flow")){ |
| - //Correct units of flow values |
| - if(f[18].equalsIgnoreCase("Flow") && (f[24].equalsIgnoreCase("cm3/sec") || f[24].equalsIgnoreCase("m3/sec")) ){//convert from cm3/sec to cfs |
| - double temp = Double.parseDouble(f[23])*(1/0.0283168466); |
| - f[24] = "cfs"; |
| - f[23] = String.valueOf(temp); |
| - } |
| - if(f[18].equalsIgnoreCase("Flow") && f[24].equalsIgnoreCase("mgd") ){//convert from million gallons per day to cfs |
| - double temp = Double.parseDouble(f[23])*(1.54722865/1); |
| - f[24] = "cfs"; |
| - f[23] = String.valueOf(temp); |
| - } |
| - if(f[18].equalsIgnoreCase("Flow") && f[24].equalsIgnoreCase("cfs")){ |
| - //Pull out only the data needed to pass between sub-functions |
| - |
| - //f[1] = Station name |
| - //f[11] = date |
| - //f[23] = flow value |
| - textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]); |
| - } |
| - }else if (Characteristic.equals("Escherichia coli") || Characteristic.equals("Fecal Coliform") || |
| - Characteristic.equals("Total Coliform")){ |
| - if(f[18].equalsIgnoreCase(Characteristic) && f[24].equalsIgnoreCase("#/100ml")){ |
| - //Pull out only the data needed to pass between sub-functions |
| - |
| - //f[1] = Station name |
| - //f[11] = date |
| - //f[23] = WQ result value (in #/100ml) |
| - textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]); |
| - } |
| - }else{ |
| - //Correct units of WQ tests |
| - if(f[24].equalsIgnoreCase("ug/l")){ |
| - double temp = Double.parseDouble(f[23])*(1/1000); |
| - f[23] = String.valueOf(temp); |
| - f[24] = "mg/l"; |
| - }else if(f[24].equalsIgnoreCase("g/l")){ |
| - double temp = Double.parseDouble(f[23])*(1000/1); |
| - f[23] = String.valueOf(temp); |
| - f[24] = "mg/l"; |
| - } |
| - if(f[18].equalsIgnoreCase(Characteristic) && f[24].equalsIgnoreCase("mg/l") && !f[23].equalsIgnoreCase("Not Detected")){ |
| - //Pull out only the data needed to pass between sub-functions |
| - |
| - //f[1] = Station name |
| - //f[11] = date |
| - //f[23] = WQ result value (in mg/L) |
| - textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]); |
| - } |
| - } |
| - |
| + if(!storetData.equalsIgnoreCase("")){ |
| + //Convert WQ test values |
| + String[] columns = currentLine.split("\t"); |
| + double conversion = getWQconversion(columns[2]); |
| + double value = Double.parseDouble(columns[1]) * conversion; |
| + storetData = storetData + "\n" + columns[0] + "\t" + columns[1] + "\t" + String.valueOf(value); |
| + }else{ |
| + storetData = currentLine; |
| } |
| - ctr++; |
| } |
| textReader.close(); |
| fr.close(); |
| - |
| - |
| - //convert Array list into String[][] array (column1 = date, column2 = value) |
| - String[][] stringArray = new String[textData.size()][2]; |
| - for(int i=0; i<stringArray.length; i++){ |
| - String[] currentColumns = textData.get(i).split("\t"); |
| - //currentLine[0] = stationID |
| - //currentLine[1] = date |
| - //currentLine[2] = value |
| - |
| - //If STORET date includes time (yyyy-mm-dd hh:mm:ss) substring out only the date |
| - if(currentColumns[1].length() > 10){ |
| - currentColumns[1] = currentColumns[1].substring(0,10); |
| - } |
| - stringArray[i][0] = currentColumns[1]; |
| - stringArray[i][1] = currentColumns[2]; |
| - } |
| - return stringArray; |
| + |
| + //Parse result data |
| + User_Data user_Data = new User_Data(); |
| + String[][] flowData = user_Data.readUserFile("STORET", stationID, storetData, wq_test, beginDate, endDate); |
| + |
| + //add check here for wq vs flow data and make sure to parse/convert the storet data's units into usgs code units |
| + return flowData; |
| } |
| /** |
| - * Opens the file and returns everything |
| - * @param partialPath the folder path to the file (ex. "C:/temp/myFolder/") |
| - * @param fileName the name of the file (ex "myFile.txt") |
| - * @return an ArrayList<String> containing all of the rows in the file |
| - * @throws IOException |
| + * @param units the units of the current USGS water quality test. |
| + * @return a double with the correct conversion factor for the units. |
| */ |
| - public ArrayList<String> OpenAllFile(String partialPath, String fileName) throws IOException { |
| - //Open file |
| - FileReader fr = new FileReader(partialPath + fileName); |
| - BufferedReader textReader = new BufferedReader(fr); |
| - |
| - ArrayList<String> textData = new ArrayList<String>(); |
| - String currentLine = ""; |
| - while((currentLine = textReader.readLine()) !=null){ |
| - textData.add(currentLine); |
| - } |
| - textReader.close(); |
| - fr.close(); |
| - |
| - return textData; |
| - } |
| - /** |
| - * Unzips the downloaded STORET zip file and extracts the contents into an ArrayList<String> for download purposes |
| - * @param zipLocation The location on the drive where the extracted text file from the downloaded STORET zip file was saved |
| - * @return an ArrayList<String> containing the downloaded datas |
| - * @throws IOException |
| - */ |
| - public ArrayList<String> Unzip_STORETDownloadFilesAll(String zipLocation) throws IOException{ |
| - //Create file names of other unzipped files |
| - String partialPath = zipLocation.substring(0,zipLocation.indexOf("tmd")); |
| - String data_location = zipLocation.substring(zipLocation.indexOf("tmd"),zipLocation.indexOf(".zip")); |
| - String data_location2 = "Data_" + data_location + "_Metadata.txt"; |
| - data_location = "Data_" + data_location + "_RegResults.txt"; |
| - |
| - System.out.println("Unzipping STORET result files"); |
| - UnZip_file(zipLocation); |
| - System.out.println("Completed unzipping STORET result files"); |
| - |
| - ArrayList<String> results = new ArrayList<String>(); |
| - System.out.println("Reading STORET result file for WQ characteristic"); |
| - |
| - //Extract "reg_results" file and extract data WQ from this combined file |
| - results = OpenAllFile(partialPath, data_location); |
| - |
| - //Delete text files and zip file |
| - System.out.println("deleting excess files"); |
| - File zip = new File(zipLocation); |
| - File text1 = new File(partialPath + data_location); |
| - File text2 = new File(partialPath + data_location2); |
| - boolean zipTF = zip.delete(); |
| - boolean text1TF = text1.delete(); |
| - boolean text2TF = text2.delete(); |
| - if(!zipTF || !text1TF || !text2TF){ |
| - System.out.println("One of the downloaded water quality files was not deleted properly"); |
| - } |
| - |
| - return results; |
| - } |
| - /** |
| - * Unzips the downloaded STORET zip file and extracts the contents into the |
| - * extpected list for LDC/FDC based on the provided characteristic |
| - * @param zipLocation The location on the drive where the extracted text |
| - * file from the downloaded STORET zip file was saved |
| - * @param characteristic The characteristic currently searched for in the |
| - * text file, either flow or the user-defined characteristic |
| - * @param deleteTrue a boolean that if true deletes the downloaded files, |
| - * otherwise keeps them (this is to delete the files for the CFA models |
| - * except LDC which has 1 download for flow and WQ and needs to keep the |
| - * files until the second dataset is read out) |
| - * @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = value |
| - * @throws IOException |
| - */ |
| - public String[][] Unzip_STORETDownloadFiles(String zipLocation, String characteristic, boolean deleteTrue) throws IOException { |
| - //Create file names of other unzipped files |
| - String partialPath = zipLocation.substring(0,zipLocation.indexOf("tmd")); |
| - String data_location = zipLocation.substring(zipLocation.indexOf("tmd"),zipLocation.indexOf(".zip")); |
| - String data_location2 = "Data_" + data_location + "_Metadata.txt"; |
| - String data_location3 = "Data_" + data_location + "_ActivityGroups.txt"; |
| - data_location = "Data_" + data_location + "_RegResults.txt"; |
| - |
| - System.out.println("Unzipping STORET result files"); |
| - UnZip_file(zipLocation); |
| - System.out.println("Completed unzipping STORET result files"); |
| - |
| - |
| - //Extract "reg_results" file and extract data WQ or flow from this combined file |
| - System.out.println("Reading STORET result file for WQ/Flow characteristic: " + characteristic); |
| - String[][] results = OpenWQFile(partialPath, data_location, characteristic); |
| - |
| - if(deleteTrue){ |
| - //Delete text files and zip file |
| - System.out.println("deleting excess files"); |
| - File zip = new File(zipLocation); |
| - File text1 = new File(partialPath + data_location); |
| - File text2 = new File(partialPath + data_location2); |
| - File text3 = new File(partialPath + data_location3); |
| - boolean zipTF = zip.delete(); |
| - boolean text1TF = text1.delete(); |
| - boolean text2TF = text2.delete(); |
| - boolean text3TF = text3.delete(); |
| - if(!zipTF || !text1TF || !text2TF || !text3TF){ |
| - System.out.println("One of the downloaded water quality files was not deleted properly"); |
| - } |
| - } |
| - return results; |
| - } |
| + private double getWQconversion(String units){ |
| + double conversion = -1.0; |
| + if(units.equalsIgnoreCase("g/l")){ |
| + conversion = 1000.0;//convert to mg/l |
| + }else if(units.equalsIgnoreCase("mg/l")){ |
| + conversion = 1.0;//convert to mg/l |
| + }else if(units.equalsIgnoreCase("ug/l")){ |
| + conversion = 0.001;//convert to mg/l |
| + }else if(units.equalsIgnoreCase("ng/l")){ |
| + conversion = (1.0/1000000.0);//convert to mg/l |
| + }else if(units.equalsIgnoreCase("pg/l")){ |
| + conversion = (1.0/1000000000.0);//convert to mg/l |
| + }else if(units.equalsIgnoreCase("cfs")){ |
| + conversion = 1.0;//convert to cfs |
| + }else if(units.equalsIgnoreCase("mgd")){ |
| + conversion = 1.54722865;//convert to cfs |
| + } |
| + return conversion; |
| + } |
| } |