Data.java [src/java/cfa] Revision: 742ac16335c3fadf896dcd0edfc0e78b7f588321 Date: Mon Jan 12 16:20:25 MST 2015
package cfa;
import java.io.IOException;
import java.util.ArrayList;
/**
* Last Updated: 12-January-2015
* @author Tyler Wible
* @since 25-January-2014
*/
public class Data {
/**
* Main Data extraction for daily flow data from the various databases that this tool can access
* @param mainFolder the output file location (used by STORET Data extraction)
* @param database the database from which to extract daily flow data (USGS, UserData, STORET, or CDWR)
* @param organizationName the organization which provided the data to the database (used by STORET Data extraction)
* @param stationID the station ID for which flow data is desired
* @param beginDate the begin date of desired flow data (yyyy-MM-dd)
* @param endDate the end date of desired flow data (yyyy-MM-dd)
* @param userData a concatenated string of User Data (tab-delimited) to extract flow data from (column1 = date, column2 = value)
* @return a String[][] of all the flow data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
* @throws Exception
*/
public String[][] extractFlowData(String mainFolder,
String database,
String organizationName,
String stationID,
String beginDate,
String endDate,
String userData) throws Exception{
//Depending on the provided inputs, search for and return flow data
String[][] flowData = new String[0][2];
if(database.equalsIgnoreCase("USGS")){
//Search for USGS flow data
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray = usgs_Data.getUSGSflowData(stationID, beginDate, endDate);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
flowData = (String[][]) returnArray[1];
//String start = (String) returnArray[2];
//String end = (String) returnArray[3];
//If there is minimal flow data, extract discharge data from the water quality database
if(flowData.length < 10){
//Retrieve all WQ data from USGS website
Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
String[][] allWQdata = (String[][]) returnArray1[1];
//String start = (String) returnArray1[2];
//String end = (String) returnArray1[3];
//Extract and combine USGS discharge water quality codes with the flow dataset
flowData = usgs_Data.getUSGSwqFlowData(flowData, allWQdata, beginDate, endDate);
}
}else if(database.equalsIgnoreCase("UserData")){
//Find the user uploaded data file and uses this for a timeseries graph
User_Data user_Data = new User_Data();
flowData = user_Data.readUserFile(userData, "flow", beginDate, endDate);
}else if(database.equalsIgnoreCase("STORET")){
//Search for STORET peak flow data
STORET_Data storet_Data = new STORET_Data();
String zip_location = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);
//Unzip results file and extract all flow data
flowData = storet_Data.Unzip_STORETDownloadFiles(zip_location, "flow", true);
}else if(database.equalsIgnoreCase("CDWR")){
//Search for CDWR flow data
CDWR_Data cdwr_Data = new CDWR_Data();
Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "Daily");
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
flowData = (String[][]) returnArray[1];
//String start = (String) returnArray[2];
//String end = (String) returnArray[3];
}
return flowData;
}
/**
* Main Data extraction for water quality data from the various databases that this tool can access
* @param mainFolder the output file location (used by STORET Data extraction)
* @param database the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
* @param organizationName the organization which provided the data to the database (used by STORET Data extraction)
* @param stationID the station ID for which water quality data is desired
* @param beginDate the begin date of desired water quality data (yyyy-MM-dd)
* @param endDate the end date of desired water quality data (yyyy-MM-dd)
* @param userData a concatenated string of User Data (tab-delimited) to extract water quality data from (column1 = date, column2 = value)
* @param wqTest the water quality test desired (if USGS it is the 5-digit water quality code for their database)
* @return a String[][] of all the specified water quality data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
* @throws IOException
* @throws InterruptedException
*/
public Object[] extractWQdata(String mainFolder,
String database,
String organizationName,
String stationID,
String beginDate,
String endDate,
String userData,
String wqTest) throws IOException, InterruptedException{
String[][] WQdata = new String[0][2];
String WQlabel = "??";
String graphUnits = "??";
if(database.equalsIgnoreCase("USGS")){
if(wqTest.length() > 5){
//Pull only the code portion of the WQ test
int endIndex = wqTest.lastIndexOf(", ");
if(endIndex == -1){
endIndex = wqTest.lastIndexOf("--");
}
WQlabel = wqTest.substring(11,endIndex);//cut off the "98335 " part before the test name and the units after the name
WQlabel = WQlabel.split(",")[0];
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Retrieve all WQ data from USGS website
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
String[][] allWQdata = (String[][]) returnArray1[1];
//String start = (String) returnArray1[2];
//String end = (String) returnArray1[3];
//Extract USGS water quality code for current wqTest only
WQdata = usgs_Data.minimizeUSGSWQdata(allWQdata, wqTest, beginDate, endDate);
//Get Units and conversion for current WQ test
graphUnits = usgs_Data.getUSGSwqUnits(wqTest);
}else if(database.equalsIgnoreCase("UserData")){
if(wqTest.length() > 5){
//Pull only the code portion of the WQ test
int endIndex = wqTest.lastIndexOf(", ");
if(endIndex == -1){
endIndex = wqTest.lastIndexOf("--");
}
WQlabel = wqTest.substring(11,endIndex);//cut off the "98335 " part before the test name and the units after the name
WQlabel = WQlabel.split(",")[0];
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Find the user uploaded data file and uses this for a timeseries graph
User_Data user_Data = new User_Data();
WQdata = user_Data.readUserFile(userData, wqTest, beginDate, endDate);
//Use the header to get the WQ test name
USGS_Data usgs_Data = new USGS_Data();
graphUnits = usgs_Data.getUSGSwqUnits(wqTest);//Because user uploaded file headers are wqTest
}else if(database.equalsIgnoreCase("STORET")){
//Search for STORET flow data
STORET_Data storet_Data = new STORET_Data();
String zipLocation = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);
//Unzip results file and extract all flow and WQ results
WQdata = storet_Data.Unzip_STORETDownloadFiles(zipLocation, wqTest, true);
WQlabel = wqTest;
graphUnits = "mg/L";
}else if(database.equalsIgnoreCase("CDWR")){
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("There is no available water quality data available for the CDWR database. This feature is only available for stations in the USGS or STORET databases.");
writeError(errorMessage);
}
Object[] returnArray = {WQdata, graphUnits, WQlabel};
return returnArray;
}
/**
* Main Data extraction for daily flow and water quality data from the various databases that this tool can access (used for LDC and LOADEST)
* @param mainFolder the output file location (used by STORET Data extraction)
* @param database the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
* @param organizationName the organization which provided the data to the database (used by STORET Data extraction)
* @param stationID the station ID for which flow and water quality data is desired
* @param beginDate the begin date of desired flow and water quality data (yyyy-MM-dd)
* @param endDate the end date of desired flow and water quality data (yyyy-MM-dd)
* @param userData a concatenated string of User Data (tab-delimited) to extract flow (column1 = date, column2 = value)
* and water quality data (column1 = date, column2 = value) with a "$$" delimiter between the two types of data
* @param wqTest the water quality test desired (if USGS it is the 5-digit water quality code for their database)
* @return a String[][] of all the specified water quality data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
* @throws IOException
* @throws InterruptedException
*/
public Object[] extractFlow_and_WQdata(String mainFolder,
String database,
String organizationName,
String stationID,
String beginDate,
String endDate,
String userData,
String wqTest) throws IOException, InterruptedException{
String[][] flowData = new String[0][2];
String[][] WQdata = new String[0][2];
if(database.equalsIgnoreCase("USGS")){
//Search for USGS flow data
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray = usgs_Data.getUSGSflowData(stationID, beginDate, endDate);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
flowData = (String[][]) returnArray[1];
//String start = (String) returnArray[2];
//String end = (String) returnArray[3];
//Retrieve all WQ data from USGS website
Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
String[][] allWQdata = (String[][]) returnArray1[1];
//String start = (String) returnArray1[2];
//String end = (String) returnArray1[3];
if(wqTest.length() > 5){
//Pull only the code portion of the WQ test
int endIndex = wqTest.lastIndexOf(", ");
if(endIndex == -1){
endIndex = wqTest.lastIndexOf("--");
}
String WQlabel = wqTest.substring(11,endIndex);//cut off the "98335 " part before the test name and the units after the name
WQlabel = WQlabel.split(",")[0];
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Extract USGS water quality code for current wqTest only
WQdata = usgs_Data.minimizeUSGSWQdata(allWQdata, wqTest, beginDate, endDate);
//Extract and combine USGS discharge water quality codes with the flow dataset
flowData = usgs_Data.getUSGSwqFlowData(flowData, allWQdata, beginDate, endDate);
}else if(database.equalsIgnoreCase("UserData")){
//Find the user uploaded data file and uses this for a timeseries graph
User_Data user_Data = new User_Data();
Object[] returnArray = user_Data.readUserFileLDC(userData, wqTest, beginDate, endDate);
flowData = (String[][]) returnArray[0];
WQdata = (String[][]) returnArray[1];
}else if(database.equalsIgnoreCase("STORET")){
//Search for STORET flow data
STORET_Data storet_Data = new STORET_Data();
String zipLocation = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "all", beginDate, endDate);
//Unzip results file and extract all flow and WQ results
flowData = storet_Data.Unzip_STORETDownloadFiles(zipLocation, "flow", false);
WQdata = storet_Data.Unzip_STORETDownloadFiles(zipLocation, wqTest, true);
}else if(database.equalsIgnoreCase("UserData")){
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("There is no available water quality data available for the CDWR database. This feature is only available for stations in the USGS or STORET databases.");
writeError(errorMessage);
}
Object[] returnArray = {flowData, WQdata};
return returnArray;
}
/**
* Main Data extraction for flood flow data from the various databases that this tool can access
* @param mainFolder the output file location (used by STORET Data extraction)
* @param database the database from which to extract daily flow data (USGS, UserData, STORET, or CDWR)
* @param organizationName the organization which provided the data to the database (used by STORET Data extraction)
* @param stationID the station ID for which flow data is desired
* @param beginDate the begin date of desired flow data (yyyy-MM-dd)
* @param endDate the end date of desired flow data (yyyy-MM-dd)
* @param userData a concatenated string of User Data (tab-delimited) to extract flow data from (column1 = date, column2 = value)
* @return a String[][] of all the flow data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
* @throws Exception
*/
public double[][] extractFloodData(String mainFolder,
String database,
String organizationName,
String stationID,
String beginDate,
String endDate,
String userData) throws IOException, Exception{
DoubleArray doubleArray = new DoubleArray();
double[][] peakFlowData = new double[0][0];
if(database.equalsIgnoreCase("USGS")){
//Search for USGS peak flow data
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray = usgs_Data.getUSGSPeakData(stationID, beginDate, endDate);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
peakFlowData = (double[][]) returnArray[1];
//double start = (double) returnArray[2];
//double end = (double) returnArray[3];
}else if(database.equalsIgnoreCase("UserData")){
//Find the user uploaded data file and uses this for a timeseries graph
User_Data user_Data = new User_Data();
String[][] flowData = user_Data.readUserFile(userData, "flow", beginDate, endDate);
//Removed duplicate dates
flowData = doubleArray.removeDuplicateDates(flowData);
//Convert into an annual peak time series
peakFlowData = doubleArray.convertSTORETpeakData(flowData);
}else if(database.equalsIgnoreCase("STORET")){
//Search for STORET peak flow data
System.out.println("calling downloadSTORET");
STORET_Data storet_Data = new STORET_Data();
String zip_location = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);
//Unzip results file and extract all flow data
String[][] flowData = storet_Data.Unzip_STORETDownloadFiles(zip_location, "flow", true);
//Removed duplicate dates
flowData = doubleArray.removeDuplicateDates(flowData);
//Convert into an annual peak time series
peakFlowData = doubleArray.convertSTORETpeakData(flowData);
}else if(database.equalsIgnoreCase("CDWR")){
//Search for CDWR flow data
CDWR_Data cdwr_Data = new CDWR_Data();
Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "Daily");
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
String[][] flowData = (String[][]) returnArray[1];
//double start = (double) returnArray[2];
//double end = (double) returnArray[3];
//Removed duplicate dates
flowData = doubleArray.removeDuplicateDates(flowData);
//Convert into an annual peak time series
peakFlowData = doubleArray.convertSTORETpeakData(flowData);
}
return peakFlowData;
}
/**
* Extracts 15 minute data from USGS and/or User data or hourly data from CDWR (as long as the date is formatted properly) returns an error otherwise
* @param mainFolder the output file location (used by STORET Data extraction)
* @param database the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
* @param stationID the station ID for which flow and water quality data is desired
* @param beginDate the begin date of desired flow and water quality data (yyyy-MM-dd)
* @param endDate the end date of desired flow and water quality data (yyyy-MM-dd)
* @param userData a concatenated string of User Data (tab-delimited) to extract flow (column1 = date, column2 = value)
* @return a String[][] of all the specified 15-minute flow data available for the specified period (column1 = date yyyy-MM-dd HH:mm format, column2 = value)
* @throws IOException
* @throws InterruptedException
* @throws Exception
*/
public String[][] extractInstantaneousFlowData(String mainFolder,
String database,
String stationID,
String beginDate,
String endDate,
String userData) throws IOException, InterruptedException, Exception{
//Depending on the provided inputs, search for and return flow data
String[][] flowData = new String[0][2];
if(database.equalsIgnoreCase("USGS")){
//Search for USGS flow data
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray = usgs_Data.getUSGS15minFlowData(stationID, beginDate, endDate);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
flowData = (String[][]) returnArray[1];
//String start = (String) returnArray[2];
//String end = (String) returnArray[3];
}else if(database.equalsIgnoreCase("UserData")){
//Find the user uploaded data file and uses this for a timeseries graph
User_Data user_Data = new User_Data();
flowData = user_Data.read15minUserFile(userData, "flow", beginDate + " 00:00", endDate + " 23:00");
}else if(database.equalsIgnoreCase("STORET")){
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("There is no available 15-minute flow data available for the STORET database. This feature is only available for stations in the USGS NWIS database.");
writeError(errorMessage);
}else if(database.equalsIgnoreCase("CDWR")){
//Search for CDWR flow data
CDWR_Data cdwr_Data = new CDWR_Data();
Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "15-min");
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
flowData = (String[][]) returnArray[1];
//String start = (String) returnArray[2];
//String end = (String) returnArray[3];
}
return flowData;
}
/**
* Main Data extraction for stage-discharge relationship data from the various databases that this tool can access
* @param database the database from which to extract data (USGS, UserData, STORET, or CDWR)
* @param stationID the station ID for which data is desired
* @return
* @throws IOException
*/
public double[][] extractStageDischarge(String database, String stationID) throws IOException, Exception{
//Depending on the provided inputs, search for and return flow data
double[][] ratingCurve = new double[0][2];
if(database.equalsIgnoreCase("USGS")){
//Search for USGS flow data
USGS_Data usgs_Data = new USGS_Data();
Object[] returnArray = usgs_Data.getUSGSratingCurve(stationID);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
ratingCurve = (double[][]) returnArray[1];
}else if(database.equalsIgnoreCase("STORET")){
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("There is no rating curve data available for STORET stations.");
writeError(errorMessage);
}else if(database.equalsIgnoreCase("CDWR")){
//Search for USGS flow data
CDWR_Data cdwr_Data = new CDWR_Data();
Object[] returnArray = cdwr_Data.getCDWRratingCurve(stationID);
//ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
ratingCurve = (double[][]) returnArray[1];
}
return ratingCurve;
}
/**
* Writes out the error message, if any, for finding the file and then exits the program
* @param error string array to be written as each line of an error message
* @throws IOException
*/
public void writeError(ArrayList<String> error) throws IOException{
//Output data to text file
String errorContents = error.get(0);
for(int i=1; i<error.size(); i++){
errorContents = errorContents + "\n" + error.get(i);
}
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
}