gui15minTimeseries_Model.java [src/java/cfa] Revision: aaaf25bc2ea783667cc842ac628a760fffeccd75 Date: Mon Nov 09 16:00:50 MST 2015
package cfa;
import java.awt.Color;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.axis.DateAxis;
import org.jfree.chart.axis.NumberAxis;
import org.jfree.chart.axis.ValueAxis;
import org.jfree.chart.plot.XYPlot;
import org.jfree.chart.title.LegendTitle;
import org.jfree.data.general.SeriesException;
import org.jfree.data.time.Day;
import org.jfree.data.time.Minute;
import org.jfree.data.time.TimeSeries;
/**
* Last Updated: 9-November-2015
* @author Tyler Wible
* @since 23-June-2014
*/
public class gui15minTimeseries_Model {
String mainFolder = "C:/Projects/TylerWible/CodeDirectories/NetBeans/data/CFA/Timeseries";
String database = "USGS";//"CDWR";//"STORET";//"UserData";//
String organizationName = "USGS";//"Co. Division of Water Resources";//"Colorado Dept. of Public Health & Environment";//
String stationID = "06752260";//"CLAGRECO";//"000028";//
String stationName = "CACHE LA POUDRE RIVER AT FORT COLLINS, CO";//"Cache La Poudre Near Greeley";//"BIG THOMPSON R NEAR MOUTH";//
String beginDate = "";//yyyy-MM-dd
String endDate = "";//yyyy-MM-dd
String userData = "";//"Date\tFlow\n1999-04-29 00:00\t8.3\n1999-05-09 00:00\t60.2\n1999-05-29 00:00\t20.1";//
boolean mergeDatasets = false;//true;//
String mergeMethod = "user";//"public";//"max";//"average";//"min";//
//Outputs
String len = "-1";
String start = "?";
String end = "?";
String units = "?";
String dataSource = "?";
double max = -1;
double min = -1;
double range = -1;
double mean = -1;
double standardDeviation = -1;
//Gets
public File getOutputSummary() {
return new File(mainFolder, "timeseries15min_summary.csv");
}
public String getGraph() {
return "timeseries15min_graph.jpg";
}
public File getTimeseriesOutput(){
//This output file is for use with JSHighCharts
return new File(mainFolder, "timeseries15min_graph.out");
}
public String getLen() {
return len;
}
public String getStart() {
return start;
}
public String getEnd() {
return end;
}
public String getUnits() {
return units;
}
public String getDataSource(){
return dataSource;
}
public String getMax() {
return String.valueOf(max);
}
public String getMin() {
return String.valueOf(min);
}
public String getRange() {
return String.valueOf(range);
}
public String getMean() {
return String.valueOf(mean);
}
public String getStandardDeviation() {
return String.valueOf(standardDeviation);
}
//Sets
public void setMainFolder(String mainFolder) {
this.mainFolder = mainFolder;
}
public void setDatabase(String database) {
this.database = database;
}
public void setOrganizationName(String organizationName) {
this.organizationName = organizationName;
}
public void setStationID(String stationID) {
this.stationID = stationID;
}
public void setStationName(String stationName) {
this.stationName = stationName;
}
public void setBeginDate(String beginDate) {
this.beginDate = beginDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
// public void setPeriod1Begin(String period1Begin) {
// this.period1Begin = period1Begin;
// }
// public void setPeriod1End(String period1End) {
// this.period1End = period1End;
// }
// public void setPeriod2Begin(String period2Begin) {
// this.period2Begin = period2Begin;
// }
// public void setPeriod2End(String period2End) {
// this.period2End = period2End;
// }
// public void setPeriod3Begin(String period3Begin) {
// this.period3Begin = period3Begin;
// }
// public void setPeriod3End(String period3End) {
// this.period3End = period3End;
// }
// public void setMedianTF(boolean medianTF) {
// this.medianTF = medianTF;
// }
public void setUserData(String userData) {
this.userData = userData;
}
public void setMergeDatasets(boolean mergeDatasets) {
this.mergeDatasets = mergeDatasets;
}
public void setMergeMethod(String mergeMethod) {
this.mergeMethod = mergeMethod;
}
/**
* Main statistics function calls other functions to calculate each statistic value then stores the results as global variables
* @param sortedData_combined data on which statistical values are desired
*/
private void CalculateDailyStatistics(String[][] flowData) throws IOException {
//Calculate flow statistics for the entire analysis period
ArrayList<Double> allData = new ArrayList<Double>();
for(int i=0; i<flowData.length; i++){
allData.add(Double.parseDouble(flowData[i][1]));
}
String[][] resultSummary = new String[6][1];
resultSummary[0][0] = "Method";
resultSummary[1][0] = "Maximum";
resultSummary[2][0] = "Minimum";
resultSummary[3][0] = "Range";
resultSummary[4][0] = "Mean";
resultSummary[5][0] = "Standard Deviation";
resultSummary = CalculateStatistics(allData, resultSummary, "All Data " + start + " to " + end);
max = Double.parseDouble(resultSummary[1][1]);
min = Double.parseDouble(resultSummary[2][1]);
range = Double.parseDouble(resultSummary[3][1]);
mean = Double.parseDouble(resultSummary[4][1]);
standardDeviation = Double.parseDouble(resultSummary[5][1]);
//Calculate Flow statistics for each day in time period
DoubleArray doubleArray = new DoubleArray();
boolean moreDays = flowData.length > 0;
String currentDay = flowData[0][0].substring(0,10);
String finalDay = flowData[flowData.length - 1][0].substring(0,10);
while(moreDays){
//Get current year's data and calculate it's statistics
ArrayList<Double> partialData = doubleArray.getDaysData(flowData, currentDay);
resultSummary = CalculateStatistics(partialData, resultSummary, currentDay);
String nextDay = doubleArray.getDay(currentDay, 1);
if(finalDay.compareToIgnoreCase(String.valueOf(nextDay)) >= 0){
currentDay = String.valueOf(nextDay);
}else{
moreDays = false;
}
}
writeSummary(resultSummary);
}
private String[][] CalculateStatistics(ArrayList<Double> dataList, String[][] resultSummary, String header){
DoubleMath doubleMath = new DoubleMath();
DoubleArray doubleArray = new DoubleArray();
double maxVal = doubleMath.max(dataList);
double minVal = doubleMath.min(dataList);
double rangeVal = maxVal - minVal;
double aveVal = doubleMath.meanArithmetic(dataList);
double stDev = doubleMath.StandardDeviationSample(dataList);
//Append current results to summary
String[] resultArray = {header,
String.valueOf(doubleMath.round(maxVal,3)),
String.valueOf(doubleMath.round(minVal,3)),
String.valueOf(doubleMath.round(rangeVal,3)),
String.valueOf(doubleMath.round(aveVal,3)),
String.valueOf(doubleMath.round(stDev,3))};
resultSummary = doubleArray.appendcolumn_Matrix(resultSummary, resultArray);
return resultSummary;
}
/**
* Graph the time series and user data and save the resulting graph to the specified location
* @param sortedData the String[][] containing sorted data for the time series
* (column 1 = dates (yyyy-mm-dd if timeStep = "daily", yyyy-mm if
* timeStep = "monthly", yyyy if timeStep = "yearly") column 2 = value
* @param sortedData_user the String[][] containing sorted user data for the
* time series (column 1 = dates (yyyy-mm-dd if timeStep = "daily", yyyy-mm
* if timeStep = "monthly", yyyy if timeStep = "yearly") column 2 = value
* @param yAxisTitle the String label for the y axis of the graph
*/
private void createTimeseriesGraph(String[][] sortedData,
String[][] sortedData_user) throws ParseException, IOException {
//Change analysis period dates into Date objects
// SimpleDateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
// Date period1Begin_date = new Date();
// Date period1End_date = new Date();
// Date period2Begin_date = new Date();
// Date period2End_date = new Date();
// Date period3Begin_date = new Date();
// Date period3End_date = new Date();
//
// if(!period1Begin.isEmpty() && !period1End.isEmpty()){
// period1Begin_date = desiredDateFormat.parse(period1Begin);
// period1End_date = desiredDateFormat.parse(period1End);
// }
// if(!period2Begin.isEmpty() && !period2End.isEmpty()){
// period2Begin_date = desiredDateFormat.parse(period2Begin);
// period2End_date = desiredDateFormat.parse(period2End);
// }
// if(!period3Begin.isEmpty() && !period3End.isEmpty()){
// period3Begin_date = desiredDateFormat.parse(period3Begin);
// period3End_date = desiredDateFormat.parse(period3End);
// }
//Create TimeSeries graph of merged data
TimeSeries series = new TimeSeries(stationID + ": Data");
// ArrayList<Double> period1 = new ArrayList<Double>();
// ArrayList<Double> period2 = new ArrayList<Double>();
// ArrayList<Double> period3 = new ArrayList<Double>();
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String[][] graphData = new String[sortedData.length][2];
for(int i=0; i < sortedData.length; i++) {
double value = Double.parseDouble(sortedData[i][1]);
Date currentDate = desiredDateFormat.parse(sortedData[i][0]);
Minute date = new Minute(currentDate);
try{
series.add(date, value);
graphData[i][0] = sortedData[i][0];
graphData[i][1] = sortedData[i][1];
}catch(SeriesException e){
//Some times there are multiple data points for a single yyyy-MM-dd HH:mm so catch the error here and continue
graphData[i][0] = "1900-01-01 00:00";
graphData[i][1] = "-1";
}catch(IllegalArgumentException e){
//If the date is prior to 1900-01-01, JFreechart will throw and error trying to parse it into the graph.
//However, keep the data for the dynamic graph which does not have this limit
graphData[i][0] = sortedData[i][0];
graphData[i][1] = sortedData[i][1];
}
// //Check periods
// Date newDate = desiredDateFormat.parse(tmpStr);
// if(newDate.compareTo(period1Begin_date) >= 0 && newDate.compareTo(period1End_date) <= 0){
// period1.add(value);
// }else if(newDate.compareTo(period2Begin_date) >= 0 && newDate.compareTo(period2End_date) <= 0){
// period2.add(value);
// }else if(newDate.compareTo(period3Begin_date) >= 0 && newDate.compareTo(period3End_date) <= 0){
// period3.add(value);
// }
}
//Create Time Series graph of user data
TimeSeries series2 = new TimeSeries(stationID + ": User Data");
for(int i=0; i < sortedData_user.length; i++) {
double value = Double.parseDouble(sortedData_user[i][1]);
int year = Integer.parseInt(sortedData_user[i][0].substring(0,4));
if(year >= 1900){
Date currentDate = desiredDateFormat.parse(sortedData_user[i][0]);
Day date = new Day(currentDate);
series2.add(date, value);
}
}
//Output XY data for use with JHighCharts
DoubleArray doubleArray = new DoubleArray();
doubleArray.writeTimeSeries(mainFolder, graphData, "15-min", getTimeseriesOutput().getName(), false);
//Create renderer, and axis for timeseries graph
Graphing graphing = new Graphing();
XYPlot plotTime = new XYPlot();
boolean showLegend = false;
//Create user data points
if(sortedData_user.length != 0){//only show user points if it is not zero
plotTime = graphing.addTimeseriesData(plotTime, series, true, Color.blue, false, false, false, true, 0);
plotTime = graphing.addTimeseriesData(plotTime, series2, true, Color.darkGray, false, false, false, true, 1);
showLegend = true;
}else{
plotTime = graphing.addTimeseriesData(plotTime, series, true, Color.blue, false, false, false, true, 0);
}
// //Create period analysis average lines
// if(!period1Begin.isEmpty() && !period1End.isEmpty()){
// CalculateStatistics(period1,"period1");
// this.len_period1 = String.valueOf(period1.size());
// double value = mean_period1;
// String label = "Average";
// if(medianTF){
// value = median_period1;
// label = "Median";
// }
// TimeSeries periodSeries = new TimeSeries("Period 1 " + label + ": " + value + " " + units);
//
// //Set Start Point
// double d = Double.parseDouble(period1Begin.substring(8));
// double m = Double.parseDouble(period1Begin.substring(5,7));
// double y = Double.parseDouble(period1Begin.substring(0,4));
// int day = (int)d;
// int month = (int)m;
// int year = (int)y;
// Day date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// //Set End Point
// d = Double.parseDouble(period1End.substring(8));
// m = Double.parseDouble(period1End.substring(5,7));
// y = Double.parseDouble(period1End.substring(0,4));
// day = (int)d;
// month = (int)m;
// year = (int)y;
// date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// plotTime = graphing.graphTimeData(plotTime, periodSeries, true, Color.red, false, false, true, true, 2);
// showLegend = true;
// }
// if(!period2Begin.isEmpty() && !period2End.isEmpty()){
// CalculateStatistics(period2,"period2");
// this.len_period2 = String.valueOf(period2.size());
// double value = mean_period2;
// String label = "Average";
// if(medianTF){
// value = median_period2;
// label = "Median";
// }
// TimeSeries periodSeries = new TimeSeries("Period 2 " + label + ": " + value + " " + units);
//
// //Set Start Point
// double d = Double.parseDouble(period2Begin.substring(8));
// double m = Double.parseDouble(period2Begin.substring(5,7));
// double y = Double.parseDouble(period2Begin.substring(0,4));
// int day = (int)d;
// int month = (int)m;
// int year = (int)y;
// Day date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// //Set End Point
// d = Double.parseDouble(period2End.substring(8));
// m = Double.parseDouble(period2End.substring(5,7));
// y = Double.parseDouble(period2End.substring(0,4));
// day = (int)d;
// month = (int)m;
// year = (int)y;
// date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// plotTime = graphing.graphTimeData(plotTime, periodSeries, true, new Color(255, 135, 0), false, false, true, true, 3);
// showLegend = true;
// }
// if(!period3Begin.isEmpty() && !period3End.isEmpty()){
// CalculateStatistics(period3,"period3");
// this.len_period3 = String.valueOf(period3.size());
// double value = mean_period3;
// String label = "Average";
// if(medianTF){
// value = median_period3;
// label = "Median";
// }
// TimeSeries periodSeries = new TimeSeries("Period 3 " + label + ": " + value + " " + units);
//
// //Set Start Point
// double d = Double.parseDouble(period3Begin.substring(8));
// double m = Double.parseDouble(period3Begin.substring(5,7));
// double y = Double.parseDouble(period3Begin.substring(0,4));
// int day = (int)d;
// int month = (int)m;
// int year = (int)y;
// Day date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// //Set End Point
// d = Double.parseDouble(period3End.substring(8));
// m = Double.parseDouble(period3End.substring(5,7));
// y = Double.parseDouble(period3End.substring(0,4));
// day = (int)d;
// month = (int)m;
// year = (int)y;
// date = new Day(day,month,year);//day,month,year
// periodSeries.add(date, value);
//
// plotTime = graphing.graphTimeData(plotTime, periodSeries, true, Color.green, false, false, true, true, 4);
// showLegend = true;
// }
//Create Y Axis
ValueAxis rangeTime = new NumberAxis("Discharge [cfs]");
plotTime.setRangeAxis(0, rangeTime);
//Create X Axis
DateAxis domainTime = new DateAxis("Date");
domainTime.setLowerMargin(0.05);
domainTime.setUpperMargin(0.05);
plotTime.setDomainAxis(0, domainTime);
//Set extra plot preferences
plotTime = graphing.setTimeAxisPreferences(plotTime);
//Create the chart with the plot and a legend
String graphTitle = "Time Series for " + database + " Station " + stationID + "; " + stationName;
JFreeChart chart = new JFreeChart(graphTitle, graphing.titleFont, plotTime, showLegend);
//Set legend Font
if(showLegend){
LegendTitle legendTitle = chart.getLegend();
legendTitle.setItemFont(graphing.masterFont);
}
//Save resulting graph for use later
try{
String path = mainFolder + File.separator + getGraph();
ChartUtilities.saveChartAsJPEG(new File(path), chart, 1280, 800);
System.out.println("JFreeChart created properly at: " + path);
}catch(IOException e){
System.err.println("A problem occurred while trying to creating the chart.");
}
}
/**
* Writes out the error message, if any, for finding the file and then exits the program
* @param error string array to be written as each line of an error message
* @throws IOException
*/
public void writeError(ArrayList<String> error) throws IOException{
//Output data to text file
String errorContents = error.get(0);
for(int i=1; i<error.size(); i++){
errorContents = errorContents + "\n" + error.get(i);
}
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
/**
* Writes out the dynamically created stats summary of the 15min data
* @param resultsSummary string array to be written as each line and column of the csv file
* @throws IOException
*/
private void writeSummary(String[][] resultsSummary) throws IOException{
//Open a file writer for the summary of the flow statistcs
String path = mainFolder + File.separator + getOutputSummary().getName();
FileWriter newFile = new FileWriter(path, false);
PrintWriter writer = new PrintWriter(newFile);
for(int i=0; i<resultsSummary.length; i++){
String currentLine = resultsSummary[i][0];
for(int j=1; j<resultsSummary[i].length; j++){
currentLine = currentLine + "," + resultsSummary[i][j];
}
writer.printf("%s" + "\r\n", currentLine);
}
//Close file writer
newFile.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
public void run() throws IOException, InterruptedException, ParseException, Exception{
//If no date input, make it the maximum of available data
if(beginDate == null || beginDate.equalsIgnoreCase("")){
if(!database.equalsIgnoreCase("UserData")){
beginDate = "2007-10-01";
}else{
beginDate = "1850-01-01";
}
}
if(endDate == null || endDate.equalsIgnoreCase("")){
// Pull current date for upper limit of data search
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date currentDate = new Date();
endDate = desiredDateFormat.format(currentDate);
}
//Check if any flow data exists
Data data = new Data();
String[][] sortableData = data.extractInstantaneousFlowData(mainFolder, database, stationID, beginDate, endDate, userData);
//If the user wants the datasets (public and user) merged then retrieve the second dataset (user)
String[][] sortableData_user = new String[0][0];
if(mergeDatasets){
User_Data user_Data = new User_Data();
sortableData_user = user_Data.read15minUserFile(userData, "flow", beginDate, endDate);
}
//Sort the Data by date to remove duplicate date entries
DoubleArray doubleArray = new DoubleArray();
Arrays.sort(sortableData, new DateComparator());
Arrays.sort(sortableData_user, new DateComparator());
//Merge the two datasets (if user data is empty nothing will be merged)
String[][] sortedData_combined = doubleArray.mergeData(sortableData, sortableData_user, mergeMethod);
if(sortedData_combined.length == 0){
ArrayList<String> errorMessage = new ArrayList<String>();
if(sortableData.length == 0){
errorMessage.add("There is no available 15-minute (instantaneous) flow data in the " + database + " database for station '" + stationID + "' and the specified date range.");
if(database.equalsIgnoreCase("CDWR")){
errorMessage.add("The CDWR database is sensitive to the begin date used, try specifying a later begin date");
}
}
if(sortableData_user.length == 0){
errorMessage.add("There is no available 15-minute (instantaneous) uploaded data for station '" + stationID + "' and the specified date range");
}
writeError(errorMessage);
}
//Save analysis results
this.start = sortedData_combined[0][0];
this.end = sortedData_combined[sortedData_combined.length - 1][0];
//Calculate stats of data
CalculateDailyStatistics(sortedData_combined);
//Graph the timeseries data
createTimeseriesGraph(sortedData_combined, sortableData_user);
//Get today's date for the source reference
Date currentDate = new Date();
SimpleDateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String today = sourceDateFormat.format(currentDate);
if(database.equalsIgnoreCase("USGS")){
this.dataSource = "Stream flow data retrieved from the U.S. Geological Survey, National Water Information System: Web Interface. http://waterdata.usgs.gov/nwis, accessed: " + today;
}else if(database.equalsIgnoreCase("CDWR")){
this.dataSource = "Stream flow data retrieved from the Colorado Division of Water Resources, CDWR. http://www.dwr.state.co.us accessed: " + today;
}else{
this.dataSource = "Stream flow data provided by the user. Flow analysis accessed: " + today;
}
this.len = String.valueOf(sortedData_combined.length);
this.units = "cfs";
}
public static void main(String[] args) throws IOException, InterruptedException, Exception {
gui15minTimeseries_Model Timeseries15min_Model = new gui15minTimeseries_Model();
//Run model
Timeseries15min_Model.run();
}
}