FlowStatistics.java [src/java/cfa] Revision: 5149545339c5424c7bc37f3a0cd787c1c62b1060 Date: Tue Jun 17 15:32:38 MDT 2014
package cfa;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Formatter;
/**
* Last Updated: 17-June-2014
* @author Tyler Wible
* @since 29-June-2011
*/
public class FlowStatistics {
//Gets
public String getFlowStatistics_summary() {
return "flow_statistics.txt";
}
/**
* Calculates the statistics (min, max) for various numbers of consecutive
* days of daily data and other hydrologic indicators of alteration.
* Additionally, this calculates the same statistics for each year within
* the dataset
* @param mainFolder the file path where the summary will be saved
* @param flowData the String[][] containing sorted data for the time series
* (column 1 = dates (yyyy-mm-dd) column 2 = value
* @param highPercentile the percentile value for defining a "high flow pulse"
* typically greater than 75% = 0.75
* @param lowPercentile the percentile value for defining a "low flow pulse"
* typically lower than 25% = 0.25
* @param m used to calculate an "m-day-average" flow
* @param n used to calculate an "m-day-average" low flow which has a
* linearly interpolated recurrence interval of "n" years
* @param dataUnits the units of the data for output purposes
* @return an Object[] where returnArray[0] = flow value of "m-day-average
* n-year recurrence low flow"
* returnValue[1] = the error message (if any) associated with calculating
* the mQn flow
* @throws IOException
*/
public Object[] calculateAnnualStatisticsSummaries(String mainFolder,
String[][] flowData,
double highPercentile,
double lowPercentile,
int m,
int n,
String dataUnits) throws IOException{
//Open a file writer for the summary of the flow statistcs per year
String path = mainFolder + File.separator + getFlowStatistics_summary();
Formatter statsSummaryFile = new Formatter(new FileWriter(path, false));
statsSummaryFile.format("%1$46s%n", "Statistics Summary for Entire Analysis Period:");
statsSummaryFile.format("%n", "");
//Calculate total data statistics
Object[] resultArray = calculateFlowStatistics(statsSummaryFile, flowData, highPercentile, lowPercentile, m, dataUnits);//These are not annual statistics because the data can include more than 1 year
statsSummaryFile = (Formatter) resultArray[0];
//double mQnFlow_temp = (Double) resultArray[1];
//Calculate Flow statistics for each year in time period
DoubleArray doubleArray = new DoubleArray();
ArrayList<Double> mQnFlows = new ArrayList<Double>();
boolean moreYears = flowData.length > 0;
String currentYear = flowData[0][0].substring(0,4);
String finalYear = flowData[flowData.length - 1][0].substring(0,4);
while(moreYears){
//Get current year's data and calculate it's statistics
String[][] partialData = doubleArray.getYearsData(flowData, currentYear);
statsSummaryFile.format("%n", "");
statsSummaryFile.format("%n", "");
statsSummaryFile.format("%1$22s%2$4s%3$1s%n", "Statistics Summary for ", currentYear, ":");
statsSummaryFile.format("%n", "");
resultArray = calculateFlowStatistics(statsSummaryFile, partialData, highPercentile, lowPercentile, m, dataUnits);//Because partial data only contains 1 year's worth of data these are annual statistics
statsSummaryFile = (Formatter) resultArray[0];
double mQnFlow_temp = (Double) resultArray[1];
mQnFlows.add(mQnFlow_temp);
int nextYear = Integer.parseInt(currentYear) + 1;
if(finalYear.compareToIgnoreCase(String.valueOf(nextYear)) >= 0){
currentYear = String.valueOf(nextYear);
}else{
moreYears = false;
}
}
//Close file writer
System.out.println("Text File located at:\t" + path);
statsSummaryFile.close();//tcw
//Calculate mQn (7Q10) Statistics
double mQnFlow = 0;
if(m != 0 && n != 0){
int ctr = 0;
for(int i=0; i<mQnFlows.size(); i++){
if(mQnFlows.get(i) != Double.NaN){
ctr++;
}
}
String[][] mQnData = new String[ctr][2];
for(int i=0; i<mQnFlows.size(); i++){
if(mQnFlows.get(i) != Double.NaN){
mQnData[ctr][0] = String.valueOf(ctr);
mQnData[ctr][1] = String.valueOf(mQnFlows.get(i));
ctr++;
}
}
double[][] mQnRanks = doubleArray.weibullPlottingPosition(mQnData);
//Find the "n" recurrence interval and return its corresponding flow as the mQnFlow
double target = (double) n/10;
for(int i=0; i<(mQnRanks.length - 1); i++){
if(n < mQnRanks[i][0] && n > mQnRanks[i+1][0]){
//Linear interpolation for flow value for "n" recurrence interval
mQnFlow = ((target - mQnRanks[i+1][0])/(mQnRanks[i][0] - mQnRanks[i+1][0]))*(mQnRanks[i][1] - mQnRanks[i+1][1]) + mQnRanks[i+1][1];
}else if(n == mQnRanks[i][0]){
mQnFlow = mQnRanks[i][1];
}
}
}
String errorMessage = "";
if(mQnFlow == 0){
errorMessage = "There is no " + m + " consecutive day flow recorded so there was no " + m + "Q" + n + " flow value calculated.";
}
Object[] returnArray = {mQnFlow, errorMessage};
return returnArray;
}
/**
*
* @param statsSummaryFile
* @param flowData
* @param highPercentile
* @param lowPercentile
* @param m
* @param dataUnits
* @return
* @throws IOException
*/
private Object[] calculateFlowStatistics(Formatter statsSummaryFile,
String[][] flowData,
double highPercentile,
double lowPercentile,
int m,
String dataUnits) throws IOException{
//Determine statistics
DoubleMath doubleMath = new DoubleMath();
//Sort Data consecutively
Arrays.sort(flowData, new DateComparator());
//Set up monthly data arrays
ArrayList<Double> jan_data = new ArrayList<Double>();
ArrayList<Double> feb_data = new ArrayList<Double>();
ArrayList<Double> mar_data = new ArrayList<Double>();
ArrayList<Double> apr_data = new ArrayList<Double>();
ArrayList<Double> may_data = new ArrayList<Double>();
ArrayList<Double> jun_data = new ArrayList<Double>();
ArrayList<Double> jul_data = new ArrayList<Double>();
ArrayList<Double> aug_data = new ArrayList<Double>();
ArrayList<Double> sep_data = new ArrayList<Double>();
ArrayList<Double> oct_data = new ArrayList<Double>();
ArrayList<Double> nov_data = new ArrayList<Double>();
ArrayList<Double> dec_data = new ArrayList<Double>();
//Calculate 1-day statistics
ArrayList<Double> average_1Day = new ArrayList<Double>();
ArrayList<Double> diffPositive = new ArrayList<Double>();
ArrayList<Double> diffNegative = new ArrayList<Double>();
double max_1day = Double.parseDouble(flowData[0][1]);
double min_1day = Double.parseDouble(flowData[0][1]);
double oldValue = 0;
String max_1day_date = flowData[0][0];
String min_1day_date = flowData[0][0];
boolean increasing = false;
int ctr_zero = 0, ctr_rises = 0, ctr_falls = 0, ctr_reversals = 0;
String date1 = flowData[0][0];
for(int i=0; i<flowData.length; i++){
String date2 = flowData[i][0];
double month = Double.parseDouble(flowData[i][0].substring(5,7));
int month_int = (int) month;
double value = Double.parseDouble(flowData[i][1]);
average_1Day.add(value);
//Store data for monthly averages
switch (month_int) {
case 1: jan_data.add(value); break;
case 2: feb_data.add(value); break;
case 3: mar_data.add(value); break;
case 4: apr_data.add(value); break;
case 5: may_data.add(value); break;
case 6: jun_data.add(value); break;
case 7: jul_data.add(value); break;
case 8: aug_data.add(value); break;
case 9: sep_data.add(value); break;
case 10: oct_data.add(value); break;
case 11: nov_data.add(value); break;
case 12: dec_data.add(value); break;
default: break;
}
//Calculate Max
if(max_1day < value){
max_1day_date = date2;
max_1day = value;
}
//Calculate Min
if(min_1day > value){
min_1day_date = date2;
min_1day = value;
}
//Count zero flow days
if(value == 0){
ctr_zero++;
}
//Calculate difference
if(checkSubsequentDates(date1, date2)){
double diff = value - oldValue;
if(diff > 0){
diffPositive.add(diff);
}else if(diff < 0){
diffNegative.add(diff);
}
//Calculate flow reversals for consecutive days only
if(value > oldValue){
if(!increasing){
ctr_reversals++;
ctr_rises++;
increasing = true;
}
}else if(value < oldValue){
if(increasing){
ctr_reversals++;
ctr_falls++;
increasing = false;
}
}
}
//Reset yesterday's flow value
oldValue = value;
date1 = date2;
}
double mean_all = doubleMath.Average(average_1Day);
max_1day = doubleMath.round(max_1day,3);
min_1day = doubleMath.round(min_1day,3);
double mean_diffPositive = doubleMath.round(doubleMath.Average(diffPositive),3);
double mean_diffNegative = doubleMath.round(doubleMath.Average(diffNegative),3);
statsSummaryFile.format("%1$20s%n", "Maximums and Minimums");
statsSummaryFile.format("%n", "");
statsSummaryFile.format("%1$17s%2$11s%3$10.2f%n","Maximum (1-day): ", max_1day_date, max_1day);
statsSummaryFile.format("%1$17s%2$11s%3$10.2f%n","Minimum (1-day): ", min_1day_date, min_1day);
//Calculate 3-day statistics
statsSummaryFile = addMinMaxResults(statsSummaryFile, flowData, 3);
//Calculate 7-day statistics
statsSummaryFile = addMinMaxResults(statsSummaryFile, flowData, 7);
//Caluculate additional 7-day statistics
Object[] resultArray = getMdayData(flowData, 7);
ArrayList<Double> average_7day = (ArrayList<Double>) resultArray[1];
double min_7day = doubleMath.min(average_7day);
double min_7day_ave = doubleMath.round(min_7day/mean_all,3);
statsSummaryFile.format("%1$34s%2$10.2f%n","Minimum (7-day) / Annual Average: ", min_7day_ave);
//Calculate 30-day statistics
statsSummaryFile = addMinMaxResults(statsSummaryFile, flowData, 30);
//Calculate 90-day statistics
statsSummaryFile = addMinMaxResults(statsSummaryFile, flowData, 90);
//Calculate mQn (7Q10)Statistics
double min_Mday = 0;
if(m != 0){
resultArray = getMdayData(flowData, m);
ArrayList<Double> average_Mday = (ArrayList<Double>) resultArray[1];
min_Mday = doubleMath.min(average_Mday);
}
//Calculate Pulse Information
double highLimit = doubleMath.Percentile_function(average_1Day, highPercentile);
double lowLimit = doubleMath.Percentile_function(average_1Day, lowPercentile);
ArrayList<Double> highPulses = new ArrayList<Double>();
ArrayList<Double> lowPulses = new ArrayList<Double>();
int ctr_highPulse = 0, ctr_lowPulse = 0;
double duration_highPulse = 0, duration_lowPulse = 0;
boolean highPulseTF = false, lowPulseTF = false;
oldValue = 0;
date1 = "";
for(int i=0; i<flowData.length; i++){
//Check for consecutive days of flow data
String date2 = flowData[i][0];
double value = Double.parseDouble(flowData[i][1]);
if(i>0){
if(checkSubsequentDates(date1, date2)){
//Calculate high pulse information
if(value > highLimit){
if(!highPulseTF){
//If on a new highPulse add to the counters
ctr_highPulse++;
duration_highPulse = 1;
highPulseTF = true;
}else{
//If still on a highPulse, add to the counter
duration_highPulse = duration_highPulse + 1;
}
}else{
//If no longer on a highPulse, store the results, reset the counters
if(highPulseTF){
highPulses.add(duration_highPulse);
highPulseTF = false;
}
}
//Calculate low pulse information
if(value < lowLimit){
if(!lowPulseTF){
//If on a new lowPulse add to the counters
ctr_lowPulse++;
duration_lowPulse = 1;
lowPulseTF = true;
}else{
//If still on a lowPulse, add to the counter
duration_lowPulse = duration_lowPulse + 1;
}
}else{
//If no longer on a lowPulse, store the results, reset the counters
if(lowPulseTF){
lowPulses.add(duration_lowPulse);
lowPulseTF = false;
}
}
}else{
//If no longer on consecutive days, store the results (if previously on a pulse) and reset the counters
if(highPulseTF){
highPulses.add(duration_highPulse);
highPulseTF = false;
}
if(lowPulseTF){
lowPulses.add(duration_lowPulse);
lowPulseTF = false;
}
}
}
//Reset yesterday's flow value
oldValue = value;
date1 = date2;
}
//Build Flow Statistics Summary for output
highLimit = doubleMath.round(highLimit,1);
lowLimit = doubleMath.round(lowLimit,1);
double janAve = doubleMath.round(doubleMath.Average(jan_data), 3);
double febAve = doubleMath.round(doubleMath.Average(feb_data), 3);
double marAve = doubleMath.round(doubleMath.Average(mar_data), 3);
double aprAve = doubleMath.round(doubleMath.Average(apr_data), 3);
double mayAve = doubleMath.round(doubleMath.Average(may_data), 3);
double junAve = doubleMath.round(doubleMath.Average(jun_data), 3);
double julAve = doubleMath.round(doubleMath.Average(jul_data), 3);
double augAve = doubleMath.round(doubleMath.Average(aug_data), 3);
double sepAve = doubleMath.round(doubleMath.Average(sep_data), 3);
double octAve = doubleMath.round(doubleMath.Average(oct_data), 3);
double novAve = doubleMath.round(doubleMath.Average(nov_data), 3);
double decAve = doubleMath.round(doubleMath.Average(dec_data), 3);
double mean_highPulses = doubleMath.round(doubleMath.Average(highPulses), 3);
double mean_lowPulses = doubleMath.round(doubleMath.Average(lowPulses), 3);
statsSummaryFile.format("%n", "");
statsSummaryFile.format("%1$19s%n", "Totals and Averages");
statsSummaryFile.format("%n", "");
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","January Average: ", janAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","February Average: ", febAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","March Average: ", marAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","April Average: ", aprAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","May Average: ", mayAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","June Average: ", junAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","July Average: ", julAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","August Average: ", augAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","September Average: ", sepAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","October Average: ", octAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","November Average: ", novAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10.2f%3$15s%n","December Average: ", decAve, dataUnits);
statsSummaryFile.format("%1$29s%2$10s%n","Number Of Zero Flow Days: ", ctr_zero);
statsSummaryFile.format("%1$29s%2$10s%n","Number Of Flow Reversals: ", ctr_reversals);
statsSummaryFile.format("%1$29s%2$10s%n","Number Of Flow Rises: ", ctr_rises);
statsSummaryFile.format("%1$29s%2$10s%n","Number Of Flow Falls: ", ctr_falls);
statsSummaryFile.format("%1$29s%2$10s%n","Number Of High Pulses: ", ctr_highPulse);
statsSummaryFile.format("%1$27s%2$12.1f%3$15s%n","Threshold For High Pulses: ", highLimit, dataUnits);
statsSummaryFile.format("%1$27s%2$6.1f%3$5s%n","Average Duration Of High Pulses: ", mean_highPulses, " days");
statsSummaryFile.format("%1$27s%2$12s%n","Number Of Low Pulses: ", ctr_lowPulse);
statsSummaryFile.format("%1$27s%2$12.1f%3$15s%n","Threshold For Low Pulses: ", lowLimit, dataUnits);
statsSummaryFile.format("%1$27s%2$6.1f%3$5s%n","Average Duration Of Low Pulses: ", mean_lowPulses, " days");
statsSummaryFile.format("%1$26s%2$10.2f%3$15s%n","Average Positive Difference Between Consecutive Days:", mean_diffPositive, dataUnits);
statsSummaryFile.format("%1$26s%2$10.2f%3$15s%n","Average Negative Difference Between Consecutive Days:", -mean_diffNegative, dataUnits);
Object[] returnArray = {statsSummaryFile, min_Mday};
return returnArray;
}
/**
* Add min/max results out the summary statistics for the flow analysis
* @throws IOException
*/
private Formatter addMinMaxResults(Formatter statsSummaryFile, String[][] flowData, int numDays) throws IOException{
//Call analysis on data
Object[] resultArray = getMdayData(flowData, numDays);
ArrayList<String> average_Mday_date = (ArrayList<String>) resultArray[0];
ArrayList<Double> average_Mday = (ArrayList<Double>) resultArray[1];
//Format results
DoubleMath doubleMath = new DoubleMath();
double max_Mday = doubleMath.max(average_Mday);
double min_Mday = doubleMath.min(average_Mday);
String max_Mday_date = getDateOfValue(average_Mday_date, average_Mday, max_Mday);
String min_Mday_date = getDateOfValue(average_Mday_date, average_Mday, min_Mday);
max_Mday = doubleMath.round(max_Mday,2);
min_Mday = doubleMath.round(min_Mday,2);
//Append results to output file
statsSummaryFile.format("%1$18s%2$26s%3$10.2f%n","Maximum (" + numDays + "-day): ", max_Mday_date + " ", max_Mday);
statsSummaryFile.format("%1$18s%2$26s%3$10.2f%n","Minimum (" + numDays + "-day): ", min_Mday_date + " ", min_Mday);
return statsSummaryFile;
}
/**
* Loops through and finds "m-day" consecutive values and takes the average of them
* @param flowData a string[][] containing: column1 = dates, column2 = flowValues
* @param numDays an integer representing the number (m) of consecutive days to be desired for analysis
* @returns an ArrayList containing an ArrayList of each set of "m-day" consecutive set of flows for analysis (min, max, average, etc)
* @throws IOException
*/
private Object[] getMdayData(String[][] flowData, int numDays) throws IOException{
//Loop through flow data and find "m"-day consecutive flows
DoubleMath doubleMath = new DoubleMath();
ArrayList<String> allDate = new ArrayList<String>();
ArrayList<Double> allData = new ArrayList<Double>();
try{
for(int i=0; i<flowData.length; i++){
ArrayList<String> mDayDate = new ArrayList<String>();
ArrayList<Double> mDayData = new ArrayList<Double>();
int ctr = i;
for(int j=0; j<numDays; j++){
if(j==0){
//Keep the first day
mDayDate.add(flowData[ctr][0]);
mDayData.add(Double.parseDouble(flowData[ctr][1]));
}else{
//Compare the current day to the previous day for consecutive-ness
boolean checkNextDate = checkSubsequentDates(flowData[ctr-1][0], flowData[ctr][0]);
if(checkNextDate){
mDayDate.add(flowData[ctr][0]);
mDayData.add(Double.parseDouble(flowData[ctr][1]));
}else{
//If not consecutive days, break out of the loop and move to the next date for flowData
mDayDate.clear();
mDayData.clear();
i = ctr - 1;//Skip to newest date since there is a break in the consecutive day data
break;
}
}
ctr++;
}
if(mDayData.size() == numDays){
//Add this m-consecutive day set of data to the all data array list for statistics later
String startDate = mDayDate.get(0);
String endDate = mDayDate.get(numDays - 1);
allDate.add(startDate + " to " + endDate);
allData.add(doubleMath.Average(mDayData));
}
}
}catch(ArrayIndexOutOfBoundsException e){
//If the consecutive day counter (ctr) goes beyond the length of data available,
//stop the subroutine and return the existing results
Object[] returnArray = {allDate, allData};
return returnArray;
}
Object[] returnArray = {allDate, allData};
return returnArray;
}
/**
* Checks if the provided dates are subsequent dates, aka nextDate = date + 1day.
* This check includes December 31st to January 1st catchs, 4-year leap-year catches,
* 100-year non-leap-year catches and 400-year leap-year catches
* @param date the first date to be compared (expected format = yyyy-mm-dd)
* @param nextDate the second date to be compared (expected format = (yyyy-mm-dd)
* @return returns true if nextDate = date + 1day, false otherwise
*/
private boolean checkSubsequentDates(String date, String nextDate){
//Date
double year = Double.parseDouble(date.substring(0,4));
double month = Double.parseDouble(date.substring(5,7));
double day = Double.parseDouble(date.substring(8));
//Next Date
double year2 = Double.parseDouble(nextDate.substring(0,4));
double month2 = Double.parseDouble(nextDate.substring(5,7));
double day2 = Double.parseDouble(nextDate.substring(8));
boolean subsequentDates = false;
//Check if nextDate = date + 1 day
if(Double.compare(year,year2) == 0){//Check if same year
if(Double.compare(month, month2) == 0){//Check if same month
if(Double.compare(day + 1, day2) == 0){//Check if subsequent day
subsequentDates = true;
}
}else{
if((Double.compare(month + 1, month2) == 0) && //Check if subsequent month
(Double.compare(day2,1) == 0)){//Check if first day
//Check months with 31 days
if((Double.compare(day, 31) == 0) &&
((Double.compare(month, 1) == 0) ||
(Double.compare(month, 3) == 0) ||
(Double.compare(month, 5) == 0) ||
(Double.compare(month, 7) == 0) ||
(Double.compare(month, 8) == 0) ||
(Double.compare(month, 10) == 0))){
subsequentDates = true;
//Check months with 30 days
}else if((Double.compare(day, 30) == 0) &&
((Double.compare(month, 4) == 0) ||
(Double.compare(month, 6) == 0) ||
(Double.compare(month, 9) == 0) ||
(Double.compare(month, 11) == 0))){
subsequentDates = true;
//Check February for leap years (including the 100-year-not-leap-year and 400-year-leap-year)
}else if((Double.compare(month, 2) == 0)){
boolean leapYear = false;
double yearUp4 = Math.ceil(year/4);
double yearDown4 = Math.floor(year/4);
double yearUp100 = Math.ceil(year/100);
double yearDown100 = Math.floor(year/100);
double yearUp400 = Math.ceil(year/400);
double yearDown400 = Math.floor(year/400);
if(yearUp400 == yearDown400){
leapYear = true;
}else if(yearUp100 == yearDown100){
leapYear = false;
}else if(yearUp4 == yearDown4){
leapYear = true;
}
//Check non-leap years (28 day February)
if(!leapYear && (Double.compare(day, 28) == 0)){//Check if is subsequent day
subsequentDates = true;
//Check leap years (29 day February)
}else if(leapYear && (Double.compare(day, 29) == 0)){//Check if is subsequent day
subsequentDates = true;
}
}
}
}
}else{
//Check if subsequent years, months, and days from December 31st to January 1st
if((Double.compare(year + 1, year2) == 0) && //Check if subsequent years
(Double.compare(month, 12) == 0) && //and the first date is December
(Double.compare(month2, 1) == 0) && //and the second date is January
(Double.compare(day, 31) == 0) && //and the first date is the 31st
(Double.compare(day2, 1) == 0)){ //and the second date is the 1st
subsequentDates = true;
}
}
return subsequentDates;
}
private String getDateOfValue(ArrayList<String> dates, ArrayList<Double> data, double value){
String dateOfValue = "null";
for(int i=0; i<dates.size(); i++){
if(data.get(i) == value){
dateOfValue = dates.get(i);
break;
}
}
return dateOfValue;
}
}