Utils.java [src/java/oms/utils] Revision: c3608cd9fa04d374f175f2ceb816faba0514620a Date: Wed Mar 25 20:08:47 MDT 2020
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package oms.utils;
import csip.Executable;
import csip.PayloadParameter;
import csip.ServiceException;
import csip.ServiceResources;
import csip.SessionLogger;
import csip.utils.Binaries;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import oms3.Conversions;
import oms3.ObjectiveFunction;
import oms3.io.CSTable;
import oms3.io.DataIO;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang.ArrayUtils;
/**
*
* @author od
*/
public class Utils {
// parameter keys
public static final String KEY_LOGLEVEL = "loglevel";
public static final String KEY_OPTIONS = "java.options";
public static final String ID_AGES_JAR = "ages.jar";
/**
*
*/
public static class PBIAS implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double diffsum = 0;
double obssum = 0;
for (int i = 0; i < sim.length; i++) {
if (obs[i] > missing) {
diffsum += sim[i] - obs[i];
obssum += obs[i];
}
}
return (diffsum / obssum) * 100.0;
}
@Override
public boolean positiveDirection() {
return false;
}
}
/**
*
*/
public static class NSLOG implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
int pre_size = sim.length;
int val_size = obs.length;
int steps = 0;
double sum_log_pd = 0;
double sum_log_vd = 0;
/**
* checking if both data arrays have the same number of elements
*/
if (pre_size != val_size) {
return missing;
} else {
steps = pre_size;
}
/**
* calculating logarithmic values of both data sets. Sets 0 if data is 0
*/
double[] log_preData = new double[pre_size];
double[] log_valData = new double[val_size];
for (int i = 0; i < steps; i++) {
if (sim[i] >= 0 && obs[i] >= 0) {
if (sim[i] == 0) {
log_preData[i] = 0;
} else {
log_preData[i] = Math.log(sim[i]);
}
if (obs[i] == 0) {
log_valData[i] = 0;
} else {
log_valData[i] = Math.log(obs[i]);
}
}
}
/**
* summing up both data sets
*/
for (int i = 0; i < log_valData.length; i++) {
sum_log_pd += log_preData[i];
sum_log_vd += log_valData[i];
}
/**
* calculating mean values for both data sets
*/
double mean_log_pd = sum_log_pd / log_valData.length;
double mean_log_vd = sum_log_vd / log_valData.length;
/**
* calculating mean pow deviations
*/
double pd_log_vd = 0;
double vd_log_mean = 0;
for (int i = 0; i < log_valData.length; i++) {
pd_log_vd = pd_log_vd + (Math.pow(Math.abs(log_valData[i] - log_preData[i]), 2));
vd_log_mean = vd_log_mean + (Math.pow(Math.abs(log_valData[i] - mean_log_vd), 2));
}
/**
* calculating efficiency after Nash & Sutcliffe (1970)
*/
double log_efficiency = 1 - (pd_log_vd / vd_log_mean);
return log_efficiency;
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
*
*/
public static class NSLOG1P implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
/**
* calculating logarithmic values of both data sets. Sets 0 if data is 0
*/
double[] log_sim = new double[sim.length];
double[] log_obs = new double[sim.length];
int valid = 0;
double sum_log_obs = 0.0;
for (int i = 0; i < sim.length; i++) {
//either prediction or validation shows a negative value.
//in this case the pair is excluded from the further calculation,
//simply by setting the values to -1 and not increasing valid pairs
// this will also handle missing values
if (sim[i] < 0.0 || obs[i] < 0.0) {
log_sim[i] = -1;
log_obs[i] = -1;
continue;
}
//both prediction and validation are equal or greater than zero
//no problem for the calculation
log_sim[i] = Math.log1p(sim[i]);
log_obs[i] = Math.log1p(obs[i]);
// summing up
sum_log_obs += log_obs[i];
valid++;
}
// calculating mean
double mean_log_obs = sum_log_obs / valid;
// calculating mean pow deviations
double sum1 = 0.0;
double sum_mean = 0.0;
for (int i = 0; i < sim.length; i++) {
if (log_sim[i] >= 0) {
sum1 += Math.pow(Math.abs(log_obs[i] - log_sim[i]), 2);
sum_mean += Math.pow(Math.abs(log_obs[i] - mean_log_obs), 2);
}
}
return 1 - (sum1 / sum_mean);
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
* create a 'sim' include file for the run part.
*/
public static void createParamInclude(Map<String, String> p,
File file) throws IOException {
StringBuilder b = new StringBuilder();
b.append("parameter {\n");
p.keySet().forEach((name) -> {
b.append(" ").append(name).append(" ").append(p.get(name)).append("\n");
});
b.append("}\n");
FileUtils.writeStringToFile(file, b.toString());
}
/**
* pass a required parameter, quoted (string).
*/
public static void passReqQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
/**
* pass optional parameter, no quotes.
*/
public static void passOptParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, param.getString(name));
}
}
}
public static void passOptQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
}
/**
* Run Ages
*
* @param dsl
* @param options
* @throws Exception
*/
public static void runAges(File dsl, File ws, PayloadParameter param,
ServiceResources res, SessionLogger LOG) throws Exception {
// Create/execute a Ages.
Executable p = createProcess(dsl, ws, param, res, LOG);
int result = p.exec();
if (result != 0) {
FilenameFilter ff = new WildcardFileFilter("java*stderr.txt", IOCase.INSENSITIVE);
File[] f = ws.listFiles(ff);
if (f != null && f.length > 0) {
String err = FileUtils.readFileToString(f[0]);
LOG.info("Ages execution error. " + f[0] + ":\n" + err);
throw new ServiceException("Ages execution error. " + f[0] + ":\n" + err);
}
throw new ServiceException("Ages execution error." + result);
}
}
/**
* Create the external Ages process.
*/
public static Executable createProcess(File dsl, File ws,
PayloadParameter param, ServiceResources res, SessionLogger LOG) throws Exception {
Map<String, String> sysprops = new HashMap();
sysprops.put("oms_prj", ws.toString());
sysprops.put("csip_ages", res.getFile(ID_AGES_JAR).getParent());
String[] jvmOptions = Binaries.asSysProps(sysprops);
String options = param.getString(KEY_OPTIONS, "");
if (options != null && !options.isEmpty()) {
jvmOptions = (String[]) ArrayUtils.addAll(jvmOptions, options.split("\\s+"));
}
// java -Doms_prj=. -cp "dist/AgES.jar" oms3.CLI -l OFF -r "projects/sfir30/simulation/sfir30.sim"
return Binaries.getResourceOMSDSL(
dsl, // the dsl file to run
jvmOptions, // jvm options
ws, // workspace dir
Arrays.asList(res.getFile(ID_AGES_JAR)), // the ages jar file
param.getString(KEY_LOGLEVEL, "INFO"), // The log level
LOG); // This session logger
}
private static final int FILE = 0;
private static final int TABLE = 1;
private static final int COLUMN = 2;
/**
*
* @param d
* @param workspace
* @param start
* @param end
* @return
* @throws IOException
*/
public static double[] getData(String d, File workspace,
String start, String end) throws IOException {
if (d == null) {
throw new IllegalArgumentException("Missing data property: " + d);
}
String[] parts = DataIO.parseCsvFilename(d);
if (parts.length != 3) {
throw new IllegalArgumentException("invalid: " + d + " expected:: <file>/<table>/<column>");
}
// System.out.println(Arrays.toString(parts));
CSTable t = DataIO.table(new File(workspace, parts[FILE]), parts[TABLE]);
Date startDate = Conversions.convert(start, Date.class);
Date endDate = Conversions.convert(end, Date.class);
// System.out.println(startDate);
// System.out.println(endDate);
double[] vals = DataIO.getColumnDoubleValuesInterval(startDate, endDate, t,
parts[COLUMN], DataIO.DAILY);
return vals;
}
// static void d() throws IOException {
// CSProperties pr = DataIO.properties(new File("/od/projects/csip-all/csip-oms/tmp/data/main_params.csv"), "Parameter");
// List<String> l = new ArrayList<>(pr.keySet()) ;
// Collections.sort(l);
// for (String string : l) {
// System.out.println("\"" + string + "\",");
// }
// }
public static void main(String[] args) throws IOException {
// small test
double[] d = getData("obs_data02_14.csv/obs/orun[1]", new File("/tmp"), "2002-01-18", "2002-02-18");
System.out.println(Arrays.toString(d));
// d();
}
}