User_Data.java [src/java/groundwater] Revision: Date:
package groundwater;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* Last Updated: 13-September-2016
* @author Tyler Wible
* @since 12-July-2012
*/
public class User_Data {
/**
* Pulls the first line of the provided file as headers and breaks it based
* on tab-delimited for use later
* @param userData_string The tab-delimited contents of the user data file
* with header The expected format of the string is a tab-delimited text
* file (\n separating lines) with dates in the first column and daily
* average cfs values in the second column (\t separating columns).
* (Acceptable date formats for the first column are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d)
* (ex. "Date\tFlow\n1999-04-29\t8.3\n1999-05-09\t60.2\n1999-05-29\t20.1")
* @return a string[] containing the header
* @throws IOException
*/
public static String[] getHeadersLDC(String userData_string) throws IOException{
String[] userData_types = userData_string.split("\\$\\$");
//String userData_flow = userData_types[0];
String userData_wq = userData_types[1];
//Pull out the first line and parse it into headers
String[] userData_wq_rows = userData_wq.split("\n");
String[] wqheaders = userData_wq_rows[0].split("\t");
return wqheaders;
}
/**
* Pulls the first line of the provided file as headers and breaks it based
* on tab-delimited for use later
* @param userData_string The tab-delimited contents of the user data file
* with header The expected format of the string is a tab-delimited text
* file (\n separating lines) with dates in the first column and daily
* average cfs values in the second column (\t separating columns).
* (Acceptable date formats for the first column are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d)
* (ex. "Date\tFlow\n1999-04-29\t8.3\n1999-05-09\t60.2\n1999-05-29\t20.1")
* @return a string[] containing the header of the first and second datasets (used by loadest and durationcurve-ldc)
* @throws IOException
*/
private static String[] getHeaders(String userData_string) throws IOException{
//Pull out the first line and parse it into headers
String[] userData_rows = userData_string.split("\n");
String[] headers = userData_rows[0].split("\t");
return headers;
}
/**
* Opens and reads out the contents of the specified file to be used as flow and water quality data.
* @param database the database from which to extract daily flow data (UserData or STORET)
* @param stationID the station ID of the data, used for error message
* @param userData_string The tab-delimited contents of the user data file
* with header The expected format of the string is a tab-delimited text
* file (\n separating lines) with dates in the first column and daily
* average cfs values in the second column (\t separating columns).
* (Acceptable date formats for the first column are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d)
* (ex. "Date\tFlow\n1999-04-29\t8.3\n1999-05-09\t60.2\n1999-05-29\t20.1")
* @param wqTest the water quality test desired "00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L" (format: "5-digit test-name -- units") or "flow"
* @param beginDate the user specified begin date, used to minimize the data returned
* @param endDate the user specified end date, used to minimize the data returned
* * @return a Object[][] containing:
* [0] = a String[][] containing flow data: column1 = year, column2 = flowValue
* [1] = a String[][] containing water quality data: column1 = year, column2 = floodValue
* @throws IOException
*/
public static Object[] readUserFileLDC(String database, String stationID, String userData_string, String wqTest, String beginDate, String endDate) throws IOException{
String[] userData_types = userData_string.split("\\$\\$");
String userData_flow = userData_types[0];
String userData_wq = userData_types[1];
String[][] flowData = readUserFile(database, stationID, userData_flow, "flow", beginDate, endDate);
String[][] wqData = readUserFile(database, stationID, userData_wq, wqTest, beginDate, endDate);
Object[] returnArray = {flowData, wqData};
return returnArray;
}
/**
* Opens and reads out the contents of the specified file to be used as flow data.
* @param database the database from which to extract daily flow data (UserData or STORET)
* @param stationID the station ID of the data, used for error message
* @param userData_string The tab-delimited contents of the user data file
* with header The expected format of the string is a tab-delimited text
* file (\n separating lines) with dates in the first column and daily
* average cfs values in the second column (\t separating columns).
* (Acceptable date formats for the first column are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d)
* (ex. "Date\tFlow\n1999-04-29\t8.3\n1999-05-09\t60.2\n1999-05-29\t20.1")
* @param wqTest the water quality test desired "00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L" (format: "5-digit test-name -- units") or "flow"
* @param beginDate the user specified begin date, used to minimize the data returned
* @param endDate the user specified end date, used to minimize the data returned
* @return a string[][] with the contents of the user uploaded file formatted as: column1 = dates (yyyy-mm-dd),
* column2 = values (expected to be daily average flow values in cfs or water quality concentrations in the units of the current water quality test)
* if there is not a value for column2 (blank, null, not a number, etc.) then no data will be kept for that date
* @throws IOException
*/
public static String[][] readUserFile(String database, String stationID, String userData_string, String wqTest, String beginDate, String endDate) throws IOException{
//Get the wqCode out of wqTest
String wqCode = wqTest;
if(wqTest.length() > 5){
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Get the headers and find the desired column of data
String[] headers = getHeaders(userData_string);
int headerIndex = -1;
for(int i=0; i<headers.length; i++){
if(headers[i].equalsIgnoreCase(wqCode)){
headerIndex = i;
}
}
//Check for lack of data
if(headerIndex == -1){
String errorContents = "There is no available uploaded " + database + " data for station " + stationID + " and water quality test:" + wqTest;
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
//Open file and read contents of the user uploaded file out into an arrayList to be parsed into the flow data
String[] userData_rows = userData_string.split("\n");
int ctr = 0;
for(int i=1; i<userData_rows.length; i++){//Skip first row header
//Check for date format problems
String[] currentColumns = userData_rows[i].split("\t");
String correctedDate = fixDateFormat(currentColumns[0]);
//currentColumns[0] = date
//currentColumns[1] = value
//Only keep the data if the value column is not null
boolean valueExists = true;
try{
double value = Double.parseDouble(currentColumns[headerIndex]);
}catch(NullPointerException | NumberFormatException | ArrayIndexOutOfBoundsException e){
valueExists = false;
}
if(!correctedDate.equalsIgnoreCase("error") && valueExists){
//If the date conversion does not hit an error keep this date
ctr++;
}
}
//Convert Array list into String[][] array (column1 = date, column2 = value)
String[][] allDataArray = new String[ctr][2];
ctr = 0;
for(int i=1; i<userData_rows.length; i++){
//Check for date format problems
String[] currentColumns = userData_rows[i].split("\t");
String correctedDate = fixDateFormat(currentColumns[0]);
//currentColumns[0] = date
//currentColumns[1] = value
//Only keep the data if the value column is not null
boolean valueExists = true;
try{
double value = Double.parseDouble(currentColumns[headerIndex]);
}catch(NullPointerException | NumberFormatException | ArrayIndexOutOfBoundsException e){
valueExists = false;
}
if(!correctedDate.equalsIgnoreCase("error") && valueExists){
//If the date conversion does not hit an error keep this date
allDataArray[ctr][0] = correctedDate;
allDataArray[ctr][1] = currentColumns[headerIndex];
ctr++;
}
}
//Reduce the data array based on the provided begin and end dates
String[][] dataArray = minimizeUserData(allDataArray, beginDate, endDate);
return dataArray;
}
/**
* Converts the provided date format into a new date format of yyyy-mm-dd which is expected by all of the java functions (supported input formats are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d)
* @param date the original date format to be converted into yyyy-mm-dd
* @return the original date converted to the yyyy-mm-dd format or "error" if the date wasn't able to be converted
*/
public static String fixDateFormat(String date){
//This allows the user to upload a greater number of date formats that will be converted into the expected format for all the java interfaces
//Convert Determine the deliminator the current date format
String[] dateColumns = date.split("-");
String deliminator = "";
String day = "01", month = "01", year = "1900";
if(dateColumns.length != 1){
deliminator = "-";
}else{
//Not a "-" separated date, so try a "/" separated date
dateColumns = date.split("/");
if(dateColumns.length != 1){
deliminator = "/";
}else{
//Not a "/" separated date, so assume that it is a number and not parse-able as a date,
//return an error so this date can be skipped
return "error";
}
}
//Substring the date
String[] datePieces = date.split(deliminator);
year = datePieces[0];
month = datePieces[1];
day = datePieces[2];
//Check if user uploaded yyyy-MM-dd HH:mm data so shorted "day" accordingly
if(day.length() > 2){
day = day.substring(0,2);
}
//Check if the date pieces are the proper sizes (a 4 digit year, 2 digit month, and 2 digit day)
if(year.length() < 4 || year.length() > 4 || month.length() > 2 || day.length() > 2){
return "error";
}
//Check for a single digit month, if so make it a 2 digit month starting with a zero
if(month.length() < 2){
month = "0" + month;
}
//Check for a single digit day, if so make it a 2 digit day starting with a zero
if(day.length() < 2){
day = "0" + day;
}
//Assemble and return the newly formatted date
String newDate = year + "-" + month + "-" + day;
return newDate;
}
/**
* Reduces all data to just that within the specified date range
* @param allData all water quality data for the earlier provided date range and station ID (column1 = date, column2 = value)
* @param beginDate the user defined begin date for data search
* @param endDate the user defined end date for data search
* @return A string array formatted the same as the input array allData (column1 = date, column2 = value) containing only the
* data for dates which were beginDate < data-date < endDate
* @throws IOException
*/
public static String[][] minimizeUserData(String[][] allData, String beginDate, String endDate) throws IOException{
//Get today's date
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date currentDate = new Date();
String todaysDate = desiredDateFormat.format(currentDate);
int ctr = 0;
for(int i=0; i<allData.length; i++){
if(todaysDate.equals(endDate)){
//If the end limit is today, keep future forcasted data as well
if((allData[i][0].compareTo(beginDate) >= 0)){
ctr++;
}
}else{
//Check if the current data is within the date range, if so keep it
if((allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
ctr++;
}
}
}
String[][] reducedData = new String[ctr][2];
ctr=0;
for(int i=0; i<allData.length; i++){
if(todaysDate.equals(endDate)){
//If the end limit is today, keep future forcasted data as well
if((allData[i][0].compareTo(beginDate) >= 0)){
reducedData[ctr][0] = allData[i][0];//date
reducedData[ctr][1] = allData[i][1];//value
ctr++;
}
}else{
//Check if the current data is within the date range, if so keep it
if((allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
reducedData[ctr][0] = allData[i][0];//date
reducedData[ctr][1] = allData[i][1];//value
ctr++;
}
}
}
return reducedData;
}
/**
* Opens and reads out the contents of the specified file to be used as flow data.
* @param userData_string The tab-delimited contents of the user data file
* with header The expected format of the string is a tab-delimited text
* file (\n separating lines) with dates in the first column and daily
* average cfs values in the second column (\t separating columns).
* (Acceptable date with military-style time formats for the first column are:
* yyyy-MM-dd,
* yyyy-MM-d,
* yyyy-M-dd,
* yyyy-M-d,
* yyyy/MM/dd,
* yyyy/MM/d,
* yyyy/M/dd,
* yyyy/M/d)
* which can be combined with any of the following time formats using a 24hr clock (military time) aka: no am/pm
* HH:mm
* HH:m
* H:mm
* H:m)
* The combined date and time format should be formatted like: yyyy-MM-dd hh:mm
* (ex. "Date\tFlow\n1999-04-29 01:30\t8.3\n1999-05-09 04:45\t60.2\n1999-05-29 12:30\t20.1")
* @param wqTest the 5-digit USGS code for a water quality test or the word "flow" used to find the correct column of the user data for analysis
* @param beginDate the user specified begin date, used to minimize the data returned
* @param endDate the user specified end date, used to minimize the data returned
* @return a string[][] with the contents of the user uploaded file formatted as: column1 = dates (yyyy-mm-dd),
* column2 = values (expected to be daily average flow values in cfs or water quality concentrations in the units of the current water quality test)
* if there is not a value for column2 (blank, null, not a number, etc.) then no data will be kept for that date
* @throws IOException
*/
public static String[][] read15minUserFile(String userData_string, String wqTest, String beginDate, String endDate) throws IOException{
//Check that the wqTest is a USGS code or "flow"
if(wqTest.length() > 5){
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Get the headers and find the desired column of data
String[] headers = getHeaders(userData_string);
int headerIndex = -1;
for(int i=0; i<headers.length; i++){
if(headers[i].equalsIgnoreCase(wqTest)){
headerIndex = i;
}
}
//Check for lack of data
if(headerIndex == -1){
String errorContents = "There is no available uploaded data for " + wqTest;
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
//Open file and read contents of the user uploaded file out into an arrayList to be parsed into the flow data
String[] userData_rows = userData_string.split("\n");
int ctr = 0;
for(int i=1; i<userData_rows.length; i++){//Skip first row header
//Check for date format problems
String[] currentColumns = userData_rows[i].split("\t");
String correctedDate = fixDateTimeFormat(currentColumns[0]);
//currentColumns[0] = date
//currentColumns[1] = value
//Only keep the data if the value column is not null
boolean valueExists = true;
try{
double value = Double.parseDouble(currentColumns[headerIndex]);
}catch(NullPointerException | NumberFormatException e){
valueExists = false;
}
if(!correctedDate.equalsIgnoreCase("error") && valueExists){
//If the date conversion does not hit an error keep this date
ctr++;
}
}
//Convert Array list into String[][] array (column1 = date, column2 = value)
String[][] allDataArray = new String[ctr][2];
ctr = 0;
for(int i=1; i<userData_rows.length; i++){
//Check for date format problems
String[] currentColumns = userData_rows[i].split("\t");
String correctedDate = fixDateTimeFormat(currentColumns[0]);
//currentColumns[0] = date
//currentColumns[1] = value
//Only keep the data if the value column is not null
boolean valueExists = true;
try{
double value = Double.parseDouble(currentColumns[headerIndex]);
}catch(NullPointerException | NumberFormatException e){
valueExists = false;
}
if(!correctedDate.equalsIgnoreCase("error") && valueExists){
//If the date conversion does not hit an error keep this date
allDataArray[ctr][0] = correctedDate;
allDataArray[ctr][1] = currentColumns[headerIndex];
ctr++;
}
}
//Reduce the data array based on the provided begin and end dates
String[][] dataArray = minimize15minUserData(allDataArray, beginDate, endDate);
return dataArray;
}
/**
* Converts the provided date-time format into a new date-time format of 'yyyy-mm-dd hh:mm' which is expected by all of the 15 minute-data java functions (supported input formats are:
* yyyy-mm-dd,
* yyyy-mm-d,
* yyyy-m-dd,
* yyyy-m-d,
* yyyy/mm/dd,
* yyyy/mm/d,
* yyyy/m/dd,
* yyyy/m/d
* which can be combined with any of the following time formats using a 24hr clock (military time) aka: no am/pm
* hh:mm
* hh:m
* h:mm
* h:m)
* The combined date and time format should be formatted like: yyyy-MM-dd hh:mm
* @param dateTime_string the original date format to be converted into yyyy-mm-dd hh:mm
* @return the original date-time converted to the 'yyyy-mm-dd hh:mm' format or "error" if the date-time wasn't able to be converted
*/
public static String fixDateTimeFormat(String dateTime_string){
//This allows the user to upload a greater number of date-time formats that will be converted into the expected format for all the java interfaces
String[] dateTime = dateTime_string.split(" ");
String newDateTime = "error";
try{
String date = dateTime[0];
String time= dateTime[1];
String newDate = fixDateFormat(date);
String newTime = fixTimeFormat(time);
if(!newDate.equalsIgnoreCase("error") || !newTime.equalsIgnoreCase("error")){
newDateTime = newDate + " " + newTime;
}
}catch(IndexOutOfBoundsException e){
return "error";
}
return newDateTime;
}
/**
* Converts the provided time format into a new time format of hh:mm which is expected by the 15-minute java functions (supported input formats are:
* hh:mm
* hh:m
* h:mm
* h:m
* @param time the original time format to be converted into hh:mm
* @return the original time converted to the hh:mm format or "error" if the time wasn't able to be converted
*/
public static String fixTimeFormat(String time){
//This allows the user to upload a greater number of time formats that will be converted into the expected format for all the java interfaces
//Substring the time
String[] timePieces = time.split(":");
if(timePieces.length != 2){
//Not a ":" separated time, so assume that it is a number and not parse-able as a time,
//return an error so this time can be skipped
return "error";
}
String hour = "00", minute = "01";
hour = timePieces[0];
minute = timePieces[1];
//Check if the date time are the proper sizes (a 2 digit hour, and 2 digit minute)
if(hour.length() > 2 || minute.length() > 2){
return "error";
}
//Check for a single digit hour, if so make it a 2 digit hour starting with a zero
if(hour.length() < 2){
hour = "0" + hour;
}
//Check for a single digit minute, if so make it a 2 digit minute starting with a zero
if(minute.length() < 2){
minute = "0" + minute;
}
//Assemble and return the newly formatted time
String newTime = hour + ":" + minute;
return newTime;
}
/**
* Reduces all data to just that within the specified date range
* @param allData all water quality data for the earlier provided date range and station ID (column1 = date, column2 = value)
* @param beginDate the user defined begin date for data search
* @param endDate the user defined end date for data search
* @return A string array formatted the same as the input array allData (column1 = date, column2 = value) containing only the
* data for dates which were beginDate < data-date < endDate
* @throws IOException
*/
public static String[][] minimize15minUserData(String[][] allData, String beginDate, String endDate) throws IOException{
//Get today's date
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date currentDate = new Date();
String todaysDate = desiredDateFormat.format(currentDate);
int ctr = 0;
for(int i=0; i<allData.length; i++){
if(todaysDate.equals(endDate)){
//If the end limit is today, keep future forcasted data as well
if((allData[i][0].compareTo(beginDate) >= 0)){
ctr++;
}
}else{
//Check if the current data is within the date range, if so keep it
if((allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
ctr++;
}
}
}
String[][] reducedData = new String[ctr][2];
ctr=0;
for(int i=0; i<allData.length; i++){
if(todaysDate.equals(endDate)){
//If the end limit is today, keep future forcasted data as well
if((allData[i][0].compareTo(beginDate) >= 0)){
reducedData[ctr][0] = allData[i][0];//date
reducedData[ctr][1] = allData[i][1];//value
ctr++;
}
}else{
//Check if the current data is within the date range, if so keep it
if((allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
reducedData[ctr][0] = allData[i][0];//date
reducedData[ctr][1] = allData[i][1];//value
ctr++;
}
}
}
return reducedData;
}
}