Utils.java [src/java/oms/utils] Revision: fb6348599b1eba4942e44374a63d93bb6633f7b3 Date: Wed Mar 25 10:18:37 MDT 2020
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package oms.utils;
import csip.Executable;
import csip.PayloadParameter;
import csip.ServiceException;
import csip.ServiceResources;
import csip.SessionLogger;
import csip.utils.Binaries;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import oms3.Conversions;
import oms3.ObjectiveFunction;
import oms3.io.CSTable;
import oms3.io.DataIO;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang.ArrayUtils;
/**
*
* @author od
*/
public class Utils {
// parameter keys
public static final String KEY_LOGLEVEL = "loglevel";
public static final String KEY_OPTIONS = "java.options";
public static final String ID_AGES_JAR = "ages.jar";
/**
*
*/
public static class PBIAS implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double diffsum = 0;
double obssum = 0;
for (int i = 0; i < sim.length; i++) {
if (obs[i] > missing) {
diffsum += sim[i] - obs[i];
obssum += obs[i];
}
}
return (diffsum / obssum) * 100.0;
}
@Override
public boolean positiveDirection() {
return false;
}
}
public static class NS2LOG implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
int pre_size = sim.length;
int val_size = obs.length;
int steps = 0;
double sum_log_pd = 0;
double sum_log_vd = 0;
/**
* checking if both data arrays have the same number of elements
*/
if (pre_size != val_size) {
return missing;
} else {
steps = pre_size;
}
/**
* calculating logarithmic values of both data sets. Sets 0 if data
* is 0
*/
double[] log_preData = new double[pre_size];
double[] log_valData = new double[val_size];
for (int i = 0; i < steps; i++) {
if (sim[i] >= 0 && obs[i] >= 0) {
if (sim[i] == 0) {
log_preData[i] = 0;
} else {
log_preData[i] = Math.log(sim[i]);
}
if (obs[i] == 0) {
log_valData[i] = 0;
} else {
log_valData[i] = Math.log(obs[i]);
}
}
}
/**
* summing up both data sets
*/
for (int i = 0; i < log_valData.length; i++) {
sum_log_pd += log_preData[i];
sum_log_vd += log_valData[i];
}
/**
* calculating mean values for both data sets
*/
double mean_log_pd = sum_log_pd / log_valData.length;
double mean_log_vd = sum_log_vd / log_valData.length;
/**
* calculating mean pow deviations
*/
double pd_log_vd = 0;
double vd_log_mean = 0;
for (int i = 0; i < log_valData.length; i++) {
pd_log_vd = pd_log_vd + (Math.pow(Math.abs(log_valData[i] - log_preData[i]), 2));
vd_log_mean = vd_log_mean + (Math.pow(Math.abs(log_valData[i] - mean_log_vd), 2));
}
/**
* calculating efficiency after Nash & Sutcliffe (1970)
*/
double log_efficiency = 1 - (pd_log_vd / vd_log_mean);
return log_efficiency;
}
@Override
public boolean positiveDirection() { // ???
return false;
}
}
/**
* create a 'sim' include file for the run part.
*/
public static void createParamInclude(Map<String, String> p,
File file) throws IOException {
StringBuilder b = new StringBuilder();
b.append("parameter {\n");
p.keySet().forEach((name) -> {
b.append(" ").append(name).append(" ").append(p.get(name)).append("\n");
});
b.append("}\n");
FileUtils.writeStringToFile(file, b.toString());
}
/**
* pass a required parameter, quoted (string).
*/
public static void passReqQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
/**
* pass optional parameter, no quotes.
*/
public static void passOptParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, param.getString(name));
}
}
}
public static void passOptQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
}
/**
* Run Ages
*
* @param dsl
* @param options
* @throws Exception
*/
public static void runAges(File dsl, File ws, PayloadParameter param,
ServiceResources res, SessionLogger LOG) throws Exception {
// Create/execute a Ages.
Executable p = createProcess(dsl, ws, param, res, LOG);
int result = p.exec();
if (result != 0) {
FilenameFilter ff = new WildcardFileFilter("java*stderr.txt", IOCase.INSENSITIVE);
File[] f = ws.listFiles(ff);
if (f != null && f.length > 0) {
String err = FileUtils.readFileToString(f[0]);
LOG.info("Ages execution error. " + f[0] + ":\n" + err);
throw new ServiceException("Ages execution error. " + f[0] + ":\n" + err);
}
throw new ServiceException("Ages execution error." + result);
}
}
/**
* Create the external Ages process.
*/
public static Executable createProcess(File dsl, File ws,
PayloadParameter param, ServiceResources res, SessionLogger LOG) throws Exception {
Map<String, String> sysprops = new HashMap();
sysprops.put("oms_prj", ws.toString());
sysprops.put("csip_ages", res.getFile(ID_AGES_JAR).getParent());
String[] jvmOptions = Binaries.asSysProps(sysprops);
String options = param.getString(KEY_OPTIONS, "");
if (options != null && !options.isEmpty()) {
jvmOptions = (String[]) ArrayUtils.addAll(jvmOptions, options.split("\\s+"));
}
// java -Doms_prj=. -cp "dist/AgES.jar" oms3.CLI -l OFF -r "projects/sfir30/simulation/sfir30.sim"
return Binaries.getResourceOMSDSL(
dsl, // the dsl file to run
jvmOptions, // jvm options
ws, // workspace dir
Arrays.asList(res.getFile(ID_AGES_JAR)), // the ages jar file
param.getString(KEY_LOGLEVEL, "INFO"), // The log level
LOG); // This session logger
}
private static final int FILE = 0;
private static final int TABLE = 1;
private static final int COLUMN = 2;
/**
*
* @param d
* @param workspace
* @param start
* @param end
* @return
* @throws IOException
*/
public static double[] getData(String d, File workspace,
String start, String end) throws IOException {
if (d == null) {
throw new IllegalArgumentException("Missing data property: " + d);
}
String[] parts = DataIO.parseCsvFilename(d);
if (parts.length != 3) {
throw new IllegalArgumentException("invalid: " + d + " expected:: <file>/<table>/<column>");
}
// System.out.println(Arrays.toString(parts));
CSTable t = DataIO.table(new File(workspace, parts[FILE]), parts[TABLE]);
Date startDate = Conversions.convert(start, Date.class);
Date endDate = Conversions.convert(end, Date.class);
// System.out.println(startDate);
// System.out.println(endDate);
double[] vals = DataIO.getColumnDoubleValuesInterval(startDate, endDate, t,
parts[COLUMN], DataIO.DAILY);
return vals;
}
// static void d() throws IOException {
// CSProperties pr = DataIO.properties(new File("/od/projects/csip-all/csip-oms/tmp/data/main_params.csv"), "Parameter");
// List<String> l = new ArrayList<>(pr.keySet()) ;
// Collections.sort(l);
// for (String string : l) {
// System.out.println("\"" + string + "\",");
// }
// }
public static void main(String[] args) throws IOException {
// small test
double[] d = getData("obs_data02_14.csv/obs/orun[1]", new File("/tmp"), "2002-01-18", "2002-02-18");
System.out.println(Arrays.toString(d));
// d();
}
}