Utils.java [src/java/oms/utils] Revision: 37a8b5cbc20db30b0d7f7e4238c495ac24efa014 Date: Wed Dec 29 18:58:50 MST 2021
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package oms.utils;
import csip.api.server.Executable;
import csip.api.server.PayloadParameter;
import csip.api.server.ServiceException;
import csip.api.server.ServiceResources;
import csip.SessionLogger;
import csip.utils.Binaries;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import oms3.Conversions;
import oms3.ObjectiveFunction;
import oms3.io.CSTable;
import oms3.io.DataIO;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
/**
*
* @author od
*/
public class Utils {
// parameter keys
public static final String KEY_LOGLEVEL = "loglevel";
public static final String KEY_OPTIONS = "java.options";
public static final String ID_AGES_JAR = "ages.jar";
/**
*
*/
public static class PBIAS implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double diffsum = 0;
double obssum = 0;
for (int i = 0; i < sim.length; i++) {
if (obs[i] > missing) {
diffsum += sim[i] - obs[i];
obssum += obs[i];
}
}
return (diffsum / obssum) * 100.0;
}
@Override
public boolean positiveDirection() {
return false;
}
}
/**
*
*/
public static class NSLOG1P implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
/**
* calculating logarithmic values of both data sets. Sets 0 if data
* is 0
*/
int valid = 0;
double avg = 0.0;
for (int i = 0; i < sim.length; i++) {
if (sim[i] >= 0.0 && obs[i] >= 0.0) {
// summing up
avg += Math.log1p(obs[i]);
valid++;
}
}
if (valid < 2) {
return Double.NEGATIVE_INFINITY;
}
// calculating mean
avg /= valid;
// calculating mean pow deviations
double rmse = 0.0;
double e = 0.0;
for (int i = 0; i < sim.length; i++) {
if (sim[i] >= 0 && obs[i] >= 0) {
double l1po = Math.log1p(obs[i]);
rmse += Math.pow(Math.abs(l1po - Math.log1p(sim[i])), 2);
e += Math.pow(Math.abs(l1po - avg), 2);
}
}
double r = 1 - (rmse / e);
return Double.isNaN(r) ? 0.0 : r;
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
*
*/
public static class NSLOG2 implements ObjectiveFunction {
@Override
public double calculate(double obs[], double sim[], double missing) {
if (obs.length != sim.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double pow = 2;
double rsme = 0;
double var = 0;
double avg = 0;
double count = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > 0 && obs[i] != missing) {
avg += Math.log(obs[i]);
count += 1;
}
}
avg /= count;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > 0 && sim[i] > 0 && obs[i] != missing) {
rsme += Math.pow(Math.abs(Math.log(obs[i]) - Math.log(sim[i])), pow);
var += Math.pow(Math.abs(Math.log(obs[i]) - avg), pow);
}
}
double result = 1.0 - (rsme / var);
if (Double.isNaN(result)) {
result = 0;
}
return result;
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
* KGE 2012
*/
public static class KGE implements ObjectiveFunction {
@Override
public double calculate(double obs[], double sim[], double missing) {
if (obs.length != sim.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
int contamedia = 0;
double sommamediaoss = 0;
double sommamediasim = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > missing) {
contamedia++;
sommamediaoss += obs[i];
sommamediasim += sim[i];
}
}
double mediaoss = sommamediaoss / contamedia;
double mediasim = sommamediasim / contamedia;
int count = 0;
double numvaprev = 0;
double coef1_den = 0;
double numR = 0;
double den1R = 0;
double den2R = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > missing) {
count++;
coef1_den += (obs[i] - mediaoss) * (obs[i] - mediaoss);
numR += (obs[i] - mediaoss) * (sim[i] - mediasim);
den1R += (obs[i] - mediaoss) * (obs[i] - mediaoss);
den2R += (sim[i] - mediasim) * (sim[i] - mediasim);
numvaprev += (sim[i] - mediasim) * (sim[i] - mediasim);
}
}
double sdosservati = Math.sqrt(coef1_den / (count - 1));
double sdsimulati = Math.sqrt(numvaprev / (count - 1));
double R = numR / (Math.sqrt(den1R) * Math.sqrt(den2R));
//double alpha = sdsimulati / sdosservati; // 2009
double beta = mediasim / mediaoss;
double gamma = (mediaoss * sdsimulati) / (sdosservati * mediasim); // 2012
//return 1 - Math.sqrt((R - 1) * (R - 1) + (alpha - 1) * (alpha - 1) + (beta - 1) * (beta - 1)); // 2009
return 1 - Math.sqrt((R - 1) * (R - 1) + (gamma - 1) * (gamma - 1) + (beta - 1) * (beta - 1)); // 2012
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
* create a 'sim' include file for the run part.
*/
public static void createParamInclude(Map<String, String> p,
File file) throws IOException {
StringBuilder b = new StringBuilder();
b.append("parameter {\n");
p.keySet().forEach((name) -> {
b.append(" ").append(name).append(" ").append(p.get(name)).append("\n");
});
b.append("}\n");
FileUtils.writeStringToFile(file, b.toString());
}
/**
* pass a required parameter, quoted (string).
*
* @param p
* @param param
* @param names
* @throws csip.api.server.ServiceException
*/
public static void passReqQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
/**
* pass optional parameter, no quotes.
*
* @param p
* @param param
* @param names
* @throws csip.api.server.ServiceException
*/
public static void passOptParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name) && param.getString(name).startsWith("-")) {
p.put(name, "\"" + param.getString(name) + "\"");
} else {
if (param.has(name) && !param.getString(name).startsWith("-")) {
p.put(name, param.getString(name));
}
}
}
}
// ArrayList<String> keyListOF = new ArrayList<>(OF.keySet());
// ArrayList<String> allList = new ArrayList<>();
// allList = keyListOF;
//
// allList.add("cal_startTime");
// allList.add("cal_endTime");
// allList.add("payload");
// allList.add("loglevel");
//
// for (String e : optParams) {
// allList.add(e);
// }
// for (String r : reqParams) {
// allList.add(r);
// }
// for (String t : flags) {
// allList.add(t);
// }
//
// // while loop
// for (String s : parameter().getNames()) {
// boolean test_in = false;
// //System.out.println(" AgES Name= " + s);
// if (allList.contains(s)) {
// test_in = true;
// }
//
// if (!test_in) {
// throw new ServiceException("Ages global parameter input error: " + s);
// //System.out.println(" not in " + s);
// }
// }
public static void passOptQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
}
/**
* Run Ages
*
* @param dsl
* @throws Exception
*/
public static void runAges(File dsl, File ws, PayloadParameter param,
ServiceResources res, SessionLogger LOG) throws Exception {
// Create/execute a Ages.
Executable p = createProcess(dsl, ws, param, res, LOG);
int result = p.exec();
if (result > 0) {
// File f = p.stderr();
// if (f.exists() && f.length() > 10) {
// String err = FileUtils.readFileToString(f, "UTF-8");
// LOG.info("Ages execution error. " + f + ":\n" + err);
// throw new ServiceException("Ages execution error. " + f + ":\n" + err);
// }
throw new ServiceException("Ages execution error." + result);
}
}
/**
* Create the external Ages process.
*/
public static Executable createProcess(File dsl, File ws,
PayloadParameter param, ServiceResources res, SessionLogger LOG) throws Exception {
Map<String, String> sysprops = new HashMap();
sysprops.put("oms_prj", ws.toString());
sysprops.put("csip_ages", res.getFile(ID_AGES_JAR).getParent());
String[] jvmOptions = Binaries.asSysProps(sysprops);
String options = param.getString(KEY_OPTIONS, "");
if (options != null && !options.isEmpty()) {
jvmOptions = (String[]) ArrayUtils.addAll(jvmOptions, options.split("\\s+"));
}
// java -Doms_prj=. -cp "dist/AgES.jar" oms3.CLI -l OFF -r "projects/sfir30/simulation/sfir30.sim"
return Binaries.getResourceOMSDSL(
dsl, // the dsl file to run
jvmOptions, // jvm options
ws, // workspace dir
Arrays.asList(res.getFile(ID_AGES_JAR)), // the ages jar file
param.getString(KEY_LOGLEVEL, "INFO"), // The log level
LOG); // This session logger
}
private static final int FILE = 0;
private static final int TABLE = 1;
private static final int COLUMN = 2;
/**
*
* @param d
* @param workspace
* @param start
* @param end
* @return
* @throws IOException
*/
public static double[] getData(String d, File workspace,
String start, String end) throws IOException {
if (d == null) {
throw new IllegalArgumentException("Missing data property: " + d);
}
String[] parts = DataIO.parseCsvFilename(d);
if (parts.length != 3) {
throw new IllegalArgumentException("invalid: " + d + " expected:: <file>/<table>/<column>");
}
// System.out.println(Arrays.toString(parts));
CSTable t = DataIO.table(new File(workspace, parts[FILE]), parts[TABLE]);
Date startDate = Conversions.convert(start, Date.class);
Date endDate = Conversions.convert(end, Date.class);
// System.out.println(startDate);
// System.out.println(endDate);
double[] vals = DataIO.getColumnDoubleValuesInterval(startDate, endDate, t,
parts[COLUMN], DataIO.DAILY);
return vals;
}
// static void d() throws IOException {
// CSProperties pr = DataIO.properties(new File("/od/projects/csip-all/csip-oms/tmp/data/main_params.csv"), "Parameter");
// List<String> l = new ArrayList<>(pr.keySet()) ;
// Collections.sort(l);
// for (String string : l) {
// System.out.println("\"" + string + "\",");
// }
// }
public static void main(String[] args) throws IOException {
// small test
//double[] d = getData("obs_data02_14.csv/obs/orun[1]", new File("/tmp"), "2002-01-18", "2002-02-18");
double[] d = getData("orun.csv/observed/orun[1]", new File("/tmp/csip/bin/ages/data/EAGLE"), "2008-01-18", "2008-02-18");
System.out.println(Arrays.toString(d));
// d();
}
}