Utils.java [src/java/oms/utils] Revision: 63d064da03658faac5472f372ab39b7a76edecba Date: Wed Aug 16 16:24:31 MDT 2023
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package oms.utils;
import csip.api.server.Executable;
import csip.api.server.PayloadParameter;
import csip.api.server.ServiceException;
import csip.api.server.ServiceResources;
import csip.SessionLogger;
import csip.utils.Binaries;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jersey.repackaged.com.google.common.base.Objects;
import oms3.Conversions;
import oms3.ObjectiveFunction;
import oms3.io.CSTable;
import oms3.io.DataIO;
import oms3.io.MemoryTable;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.codehaus.jettison.json.JSONArray;
import static m.oms.Resources.LOCATION_MNT_DATA;
/**
*
* @author od
*/
public class Utils {
// parameter keys
public static final String PROJECT = "project";
public static final String KEY_LOGLEVEL = "loglevel";
public static final String KEY_OPTIONS = "java.options";
public static final String ID_AGES_JAR = "ages.jar";
public static final String FILE_PARAMETER_REGEX_PATTERN = "\\A([^/]+)/([^/]+)/([^/]+)(?:/([^/;]+(?:;[^/;]+)*))?\\Z";
/**
*
*/
public static class PBIAS implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double diffsum = 0;
double obssum = 0;
for (int i = 0; i < sim.length; i++) {
if (obs[i] > missing) {
diffsum += sim[i] - obs[i];
obssum += obs[i];
}
}
return (diffsum / obssum) * 100.0;
}
@Override
public boolean positiveDirection() {
return false;
}
}
/**
*
*/
public static class NSLOG1P implements ObjectiveFunction {
@Override
public double calculate(double[] obs, double[] sim, double missing) {
if (sim.length != obs.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
/**
* calculating logarithmic values of both data sets. Sets 0 if data
* is 0
*/
int valid = 0;
double avg = 0.0;
for (int i = 0; i < sim.length; i++) {
if (sim[i] >= 0.0 && obs[i] >= 0.0) {
// summing up
avg += Math.log1p(obs[i]);
valid++;
}
}
if (valid < 2) {
return Double.NEGATIVE_INFINITY;
}
// calculating mean
avg /= valid;
// calculating mean pow deviations
double rmse = 0.0;
double e = 0.0;
for (int i = 0; i < sim.length; i++) {
if (sim[i] >= 0 && obs[i] >= 0) {
double l1po = Math.log1p(obs[i]);
rmse += Math.pow(Math.abs(l1po - Math.log1p(sim[i])), 2);
e += Math.pow(Math.abs(l1po - avg), 2);
}
}
double r = 1 - (rmse / e);
return Double.isNaN(r) ? 0.0 : r;
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
*
*/
public static class NSLOG2 implements ObjectiveFunction {
@Override
public double calculate(double obs[], double sim[], double missing) {
if (obs.length != sim.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
double pow = 2;
double rsme = 0;
double var = 0;
double avg = 0;
double count = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > 0 && obs[i] != missing) {
avg += Math.log(obs[i]);
count += 1;
}
}
avg /= count;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > 0 && sim[i] > 0 && obs[i] != missing) {
rsme += Math.pow(Math.abs(Math.log(obs[i]) - Math.log(sim[i])), pow);
var += Math.pow(Math.abs(Math.log(obs[i]) - avg), pow);
}
}
double result = 1.0 - (rsme / var);
if (Double.isNaN(result)) {
result = 0;
}
return result;
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
* KGE 2012
*/
public static class KGE implements ObjectiveFunction {
@Override
public double calculate(double obs[], double sim[], double missing) {
if (obs.length != sim.length) {
throw new IllegalArgumentException("obs/sim length differ: " + obs.length + "!=" + sim.length);
}
int contamedia = 0;
double sommamediaoss = 0;
double sommamediasim = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > missing) {
contamedia++;
sommamediaoss += obs[i];
sommamediasim += sim[i];
}
}
double mediaoss = sommamediaoss / contamedia;
double mediasim = sommamediasim / contamedia;
int count = 0;
double numvaprev = 0;
double coef1_den = 0;
double numR = 0;
double den1R = 0;
double den2R = 0;
for (int i = 0; i < obs.length; i++) {
if (obs[i] > missing) {
count++;
coef1_den += (obs[i] - mediaoss) * (obs[i] - mediaoss);
numR += (obs[i] - mediaoss) * (sim[i] - mediasim);
den1R += (obs[i] - mediaoss) * (obs[i] - mediaoss);
den2R += (sim[i] - mediasim) * (sim[i] - mediasim);
numvaprev += (sim[i] - mediasim) * (sim[i] - mediasim);
}
}
double sdosservati = Math.sqrt(coef1_den / (count - 1));
double sdsimulati = Math.sqrt(numvaprev / (count - 1));
double R = numR / (Math.sqrt(den1R) * Math.sqrt(den2R));
//double alpha = sdsimulati / sdosservati; // 2009
double beta = mediasim / mediaoss;
double gamma = (mediaoss * sdsimulati) / (sdosservati * mediasim); // 2012
//return 1 - Math.sqrt((R - 1) * (R - 1) + (alpha - 1) * (alpha - 1) + (beta - 1) * (beta - 1)); // 2009
return 1 - Math.sqrt((R - 1) * (R - 1) + (gamma - 1) * (gamma - 1) + (beta - 1) * (beta - 1)); // 2012
}
@Override
public boolean positiveDirection() {
return true;
}
}
/**
* create a 'sim' include file for the run part.
*/
public static void createParamInclude(Map<String, String> p,
File file) throws IOException {
StringBuilder b = new StringBuilder();
b.append("parameter {\n");
p.keySet().forEach((name) -> {
b.append(" ").append(name).append(" ").append(p.get(name)).append("\n");
});
b.append("}\n");
FileUtils.writeStringToFile(file, b.toString());
}
/**
* pass a required parameter, quoted (string).
*
* @param p
* @param param
* @param names
* @throws csip.api.server.ServiceException
*/
public static void passReqQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
/**
* pass optional parameter, no quotes.
*
* @param p
* @param param
* @param names
* @throws csip.api.server.ServiceException
*/
public static void passOptParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name) && param.getString(name).startsWith("-")) {
p.put(name, "\"" + param.getString(name) + "\"");
} else {
if (param.has(name) && !param.getString(name).startsWith("-")) {
p.put(name, param.getString(name));
}
}
}
}
// ArrayList<String> keyListOF = new ArrayList<>(OF.keySet());
// ArrayList<String> allList = new ArrayList<>();
// allList = keyListOF;
//
// allList.add("cal_startTime");
// allList.add("cal_endTime");
// allList.add("payload");
// allList.add("loglevel");
//
// for (String e : optParams) {
// allList.add(e);
// }
// for (String r : reqParams) {
// allList.add(r);
// }
// for (String t : flags) {
// allList.add(t);
// }
//
// // while loop
// for (String s : parameter().getNames()) {
// boolean test_in = false;
// //System.out.println(" AgES Name= " + s);
// if (allList.contains(s)) {
// test_in = true;
// }
//
// if (!test_in) {
// throw new ServiceException("Ages global parameter input error: " + s);
// //System.out.println(" not in " + s);
// }
// }
public static void passOptQuotedParam(Map<String, String> p,
PayloadParameter param, String... names) throws ServiceException {
for (String name : names) {
if (param.has(name)) {
p.put(name, "\"" + param.getString(name) + "\"");
}
}
}
public static void processFileParameters(PayloadParameter param,
File dataFolder, File outputFolder, Map<String, String> p) throws ServiceException {
// soils.csv/soils/aircapacity/101;102;103;104
Pattern pattern = Pattern.compile(FILE_PARAMETER_REGEX_PATTERN);
Map<String, MemoryTable> tableMap = new HashMap<>();
for (String name : param.getNames()) {
Matcher matcher = pattern.matcher(name);
if (matcher.matches()) {
String fileName = matcher.group(1);
String tableName = matcher.group(2);
String variableName = matcher.group(3);
String idListString = matcher.group(4);
String[] values = convertJSONArrayToStringArray(param.getJSONArray(name));
String tableId = fileName + "/" + tableName;
MemoryTable table;
if (tableMap.containsKey(tableId)) {
table = tableMap.get(tableId);
} else {
File file = new File(dataFolder, fileName);
if (!file.exists()) {
throw new ServiceException("File does not exist: " + file);
}
try {
table = new MemoryTable(DataIO.table(file, tableName), false);
} catch (IOException ex) {
throw new ServiceException(ex);
}
tableMap.put(tableId, table);
}
if (idListString == null) {
modifyMemoryTable(table, variableName, values);
} else {
String[] idList = idListString.split(";");
modifyMemoryTableById(table, 1, idList, variableName, values);
}
}
}
for (Map.Entry<String, MemoryTable> entry : tableMap.entrySet()) {
String[] split = entry.getKey().split("/");
String fileName = split[0];
String tableName = split[1];
File file = new File(outputFolder, fileName);
try {
DataIO.save(entry.getValue(), file);
} catch (IOException ex) {
throw new ServiceException(ex);
}
String parameterName = lookupParameterByTableName(tableName);
if (parameterName != null) {
String filePath = file.getAbsolutePath();
filePath = filePath.replace('\\', '/');
p.put(parameterName, "\"" + filePath + "\"");
}
}
}
public static String[] convertJSONArrayToStringArray(JSONArray array) {
String[] arr = new String[array.length()];
for (int i = 0; i < arr.length; ++i) {
arr[i] = array.optString(i, "");
}
return arr;
}
public static void modifyMemoryTable(MemoryTable table, String columnName, String[] values) {
int variableIndex = DataIO.columnIndex(table, columnName);
for (int i = 0; i < table.getRowCount() && i < values.length; ++i) {
table.setValueAt(values[i], i, variableIndex);
}
}
public static void modifyMemoryTableById(MemoryTable table, int idColumn, String[] idList, String columnName, String[] values) {
int variableIndex = DataIO.columnIndex(table, columnName);
for (int i = 0; i < table.getRowCount(); ++i) {
String id = String.valueOf(table.getValueAt(i, idColumn));
for (int j = 0; j < idList.length; ++j) {
if (Objects.equal(id, idList[j])) {
table.setValueAt(values[j], i, variableIndex);
break;
}
}
}
}
private static String lookupParameterByTableName(String tableName) {
switch (tableName) {
case "hrus":
return "hruFilePath";
case "landuse":
return "landuseFilePath";
case "hgeo":
return "hydroGeologyFilePath";
case "reaches":
return "reachFilePath";
case "fert":
return "fertilizerFilePath";
case "till":
return "tillageFilePath";
case "crop":
return "cropFilePath";
case "crop_upgm":
return "cropUPGMFilePath";
case "management":
return "managementFilePath";
case "soils":
return "soilFilePath";
case "routing":
return "routingFilePath";
case "irri":
return "irrigationFilePath";
case "man_irri":
return "managementIrrigationFilePath";
case "rot":
return "rotationFilePath";
case "rot_hru":
return "hruRotationFilePath";
case "hrus_additional":
return "hruAdditionalFilePath";
case "reach_additional":
return "reachAdditionalFilePath";
case "temporal_additional":
return "temporalAdditionalFilePath";
case "hru_override":
return "hruOverrideFilePath";
default:
return null;
}
}
/**
* Run Ages
*
* @param dsl
* @throws Exception
*/
public static void runAges(File dsl, File ws, PayloadParameter param,
ServiceResources res, SessionLogger LOG) throws Exception {
// Create/execute a Ages.
Executable p = createProcess(dsl, ws, param, res, LOG);
int result = p.exec();
if (result > 0) {
// File f = p.stderr();
// if (f.exists() && f.length() > 10) {
// String err = FileUtils.readFileToString(f, "UTF-8");
// LOG.info("Ages execution error. " + f + ":\n" + err);
// throw new ServiceException("Ages execution error. " + f + ":\n" + err);
// }
throw new ServiceException("Ages execution error." + result);
}
}
/**
* Create the external Ages process.
*
* @param dsl
* @param ws
* @param param
* @param res
* @param LOG
* @return
* @throws java.lang.Exception
*/
public static Executable createProcess(File dsl, File ws,
PayloadParameter param, ServiceResources res, SessionLogger LOG) throws Exception {
Map<String, String> sysprops = new HashMap();
sysprops.put("oms_prj", ws.toString());
sysprops.put("csip_ages", res.getFile(ID_AGES_JAR).getParent());
String[] jvmOptions = Binaries.asSysProps(sysprops);
String options = param.getString(KEY_OPTIONS, "");
if (options != null && !options.isEmpty()) {
jvmOptions = (String[]) ArrayUtils.addAll(jvmOptions, options.split("\\s+"));
}
// java -Doms_prj=. -cp "dist/AgES.jar" oms3.CLI -l OFF -r "projects/sfir30/simulation/sfir30.sim"
return Binaries.getResourceOMSDSL(
dsl, // the dsl file to run
jvmOptions, // jvm options
ws, // workspace dir
Arrays.asList(res.getFile(ID_AGES_JAR)), // the ages jar file
param.getString(KEY_LOGLEVEL, "INFO"), // The log level
LOG); // This session logger
}
/**
* Run Ages
*
* @param dsl
* @throws Exception
*/
public static void runAgesNew(File dsl, File ws, PayloadParameter param,
ServiceResources res, SessionLogger LOG) throws Exception {
// Create/execute a Ages.
Executable p = createProcessNew(dsl, ws, param, res, LOG);
int result = p.exec();
if (result > 0) {
// File f = p.stderr();
// if (f.exists() && f.length() > 10) {
// String err = FileUtils.readFileToString(f, "UTF-8");
// LOG.info("Ages execution error. " + f + ":\n" + err);
// throw new ServiceException("Ages execution error. " + f + ":\n" + err);
// }
throw new ServiceException("Ages execution error." + result);
}
}
/**
* Create the external Ages process.
*
* @param dsl
* @param ws
* @param param
* @param res
* @param LOG
* @return
* @throws java.lang.Exception
*/
public static Executable createProcessNew(File dsl, File ws,
PayloadParameter param, ServiceResources res, SessionLogger LOG) throws Exception {
Map<String, String> sysprops = new HashMap();
sysprops.put("oms_prj", ws.toString());
//System.out.println(" !!!! ws: " + ws.toString());
sysprops.put("csip_ages", LOCATION_MNT_DATA + "/ages_projects/ages_1_0/" + param.getString(PROJECT, "SFIR3"));
//System.out.println(" csip_ages: " + LOCATION_MNT_DATA + "/ages_projects/ages_1_0/" + param.getString(PROJECT, "SFIR3"));
String[] jvmOptions = Binaries.asSysProps(sysprops);
String options = param.getString(KEY_OPTIONS, "");
if (options != null && !options.isEmpty()) {
jvmOptions = (String[]) ArrayUtils.addAll(jvmOptions, options.split("\\s+"));
}
// java -Doms_prj=. -cp "dist/AgES.jar" oms3.CLI -l OFF -r "projects/sfir30/simulation/sfir30.sim"
return Binaries.getResourceOMSDSL(
dsl, // the dsl file to run
jvmOptions, // jvm options
ws, // workspace dir
Arrays.asList(res.getFile(ID_AGES_JAR)), // the ages jar file
param.getString(KEY_LOGLEVEL, "INFO"), // The log level
LOG); // This session logger
}
private static final int FILE = 0;
private static final int TABLE = 1;
private static final int COLUMN = 2;
/**
*
* @param d
* @param workspace
* @param start
* @param end
* @return
* @throws IOException
*/
public static double[] getData(String d, File workspace,
String start, String end) throws IOException {
if (d == null) {
throw new IllegalArgumentException("Missing data property: " + d);
}
String[] parts = DataIO.parseCsvFilename(d);
if (parts.length != 3) {
throw new IllegalArgumentException("invalid: " + d + " expected:: <file>/<table>/<column>");
}
// System.out.println(Arrays.toString(parts));
CSTable t = DataIO.table(new File(workspace, parts[FILE]), parts[TABLE]);
Date startDate = Conversions.convert(start, Date.class);
Date endDate = Conversions.convert(end, Date.class);
// System.out.println(startDate);
// System.out.println(endDate);
double[] vals = DataIO.getColumnDoubleValuesInterval(startDate, endDate, t,
parts[COLUMN], DataIO.DAILY);
return vals;
}
// static void d() throws IOException {
// CSProperties pr = DataIO.properties(new File("/od/projects/csip-all/csip-oms/tmp/data/main_params.csv"), "Parameter");
// List<String> l = new ArrayList<>(pr.keySet()) ;
// Collections.sort(l);
// for (String string : l) {
// System.out.println("\"" + string + "\",");
// }
// }
public static void main(String[] args) throws IOException {
// small test
//double[] d = getData("obs_data02_14.csv/obs/orun[1]", new File("/tmp"), "2002-01-18", "2002-02-18");
double[] d = getData("orun.csv/observed/orun[1]", new File("/tmp/csip/bin/ages/data/EAGLE"), "2008-01-18", "2008-02-18");
System.out.println(Arrays.toString(d));
// d();
}
}