R2Run.java [src/java/m/rusle2] Revision: Date:
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package m.rusle2;
import csip.Config;
import csip.Executable;
import csip.ServiceException;
import java.io.File;
import java.io.IOException;
import java.util.logging.Logger;
import oms3.util.ProcessComponent;
import csip.utils.Services;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.logging.Level;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;
/**
*
* @author od
*/
class R2Run {
public static final Logger LOG = Logger.getLogger("R2Run");
String stdout;
String stderr;
ProcessComponent pc = new ProcessComponent();
/**
* Execute the rusle2 pyrome python script
*
* We assume that Python3.4 is installed under
* /home/ubuntu/.wine/drive_c/Python34 and that wine's working dir is
* /home/ubuntu/.wine The ownership of wine's working directory must
* correspond with the user running tomcat
*
* @param r2_rsh
* @return
* @throws IOException
*/
int executePyrome(File r2_rsh, Executable python) throws IOException, ServiceException {
LOG.info("Execute method");
//String binDir = Config.getString("m.bin.dir", "/tmp/csip/bin");
//File romeshell = Binaries.unpackResource("/bin/win-x86/RomeShell.exe", new File(binDir));
//Executable python = resources().getExe(V2_0_1.PYTHON);
python.setArguments("rusle2csip.py");
LOG.info("Executing pyrome rusle2");
int ret = python.exec();
LOG.info("executed pyrome rusle2");
if (ret != 0) {
throw new ServiceException("RUSLE 2 PYTHON error: error executing pyrome:" + ret);
}
// Binaries.unpackResource("/bin/win-x86/RomeDLL.dll", new File(binDir));
// Binaries.unpackResource("/bin/win-x86/pyrome.py", new File(binDir));
// Binaries.unpackResource("/bin/win-x86/_pyrome.pyd", new File(binDir));
// File runpyrome = new File(r2_rsh.getParent() + "/runrusle2.sh");
// String runrusle2 = "winetricks vd=off ; wine /home/ubuntu/.wine/drive_c/Python34/python.exe rusle2csip.py\n";
// FileUtils.writeStringToFile(runpyrome, runrusle2);
// runpyrome.setExecutable(true);
// pc.exe = "./runrusle2.sh";
// pc.working_dir = r2_rsh.getParent();
// pc.args = new String[]{};
stdout += "\n" + FileUtils.readFileToString(python.stdout());
stderr += "\n" + FileUtils.readFileToString(python.stderr());
// LOG.info("Executing pyrome rusle2");
// pc.execute();
// LOG.info("executed pyrome rusle2");
// stdout += "\n" + pc.stdout;
// stderr += "\n" + pc.stderr;
LOG.info("stdout: " + stdout);
LOG.info(("stderr: " + stderr));
//LOG.info("rusle2 model run stdout=" + stdout);
LOG.info("exit val: " + ret);
return ret;
}
// prepares soils file for legacy 1.3 rusle2 service
int prepareSoilsFile(String soilsHttpPtr, File workingDir, boolean bPyrome, String idx) throws IOException {
// Generate shell script to prepare soils file for romeshell or pyrome
ProcessComponent pcPrepareSoils = new ProcessComponent();
File preparesoils = new File(workingDir.getParent() + "/preparesoils" + idx + ".sh");
String prepsoils = "wget \"" + soilsHttpPtr + "\" -O soils_file" + idx + ".tmp\n";
if (bPyrome) {
prepsoils += "echo \"<?xml version=\"\\\"1.0\\\"\"?>\" > soils_file" + idx + ".xml\n";
}
prepsoils += "numlines=`cat soils_file" + idx + ".tmp | wc -l`\n";
prepsoils += "if [ $numlines -eq 1 ]\n";
prepsoils += "then\n";
prepsoils += " tail -n +1 soils_file" + idx + ".tmp >> soils_file" + idx + ".xml\n";
prepsoils += "else\n";
prepsoils += " tail -n +2 soils_file" + idx + ".tmp >> soils_file" + idx + ".xml\n";
prepsoils += "fi\n";
FileUtils.writeStringToFile(preparesoils, prepsoils);
preparesoils.setExecutable(true);
LOG.info("soilsHttpPtr=" + soilsHttpPtr);
LOG.info("the pc-exe is=" + pcPrepareSoils.exe);
pcPrepareSoils.working_dir = workingDir.getParent();
pcPrepareSoils.exe = "./preparesoils" + idx + ".sh";
pcPrepareSoils.args = new String[]{};
pcPrepareSoils.execute();
stdout += "\n" + pcPrepareSoils.stdout;
stderr += "\n" + pcPrepareSoils.stderr;
return pcPrepareSoils.exitValue;
}
//// @Deprecated
// int prepareClimateFile(String climateHttpPtr, File workingDir) throws IOException
// {
// ProcessComponent pcPrepareCli = new ProcessComponent();
// // Generate shell script to prepare climate file for pyrome
// File preparecli = new File(workingDir.getParent() + "/preparecli.sh");
// String prepcli = "wget \"" + climateHttpPtr + "\" -O cli_file0.tmp\n";
// prepcli += "echo \"<?xml version=\\\"1.0\\\"?>\" > cli_file0.xml\n";
// prepcli += "echo \"<Obj>\" >> cli_file0.xml\n";
// prepcli += "echo \"<Type>CLIMATE</Type>\" >> cli_file0.xml\n";
// prepcli += "echo \"<Filename>climates\\\\\\\\aaa</Filename>\" >> cli_file0.xml\n";
// prepcli += "tail -n +2 cli_file0.tmp >> cli_file0.xml\n";
// FileUtils.writeStringToFile(preparecli, prepcli);
// preparecli.setExecutable(true);
// LOG.info("climateHttpPtr=" + climateHttpPtr);
// LOG.info("the pc-exe is=" + pcPrepareCli.exe);
// pcPrepareCli.working_dir = workingDir.getParent();
// pcPrepareCli.exe = "./preparecli.sh";
// pcPrepareCli.args = new String[]{};
// pcPrepareCli.execute();
// stdout += "\n" + pcPrepareCli.stdout;
// stderr += "\n" + pcPrepareCli.stderr;
// return pcPrepareCli.exitValue;
// }
// void prepareClimateFileJ(String climateHttpPtr, File workingDir) throws IOException
// {
// prepareFileJ(climateHttpPtr, new File(workingDir.getParent(), "cli_file0.xml"), "CLIMATE", "climates", "");
//
// }
// // Must find the hydraulic element flow path element of a hydraulic element system
// // download this file, and modify the header, so it can be fed to python
// @Deprecated
// int prepareHydraulicElementFlowPath(String hydElemPtr, String r2db, File workingDir) throws IOException
// {
// ProcessComponent pcPrepareHydElemFP = new ProcessComponent();
// File preparehydElemFP = new File(workingDir.getParent() + "/preparehydraulic-element.sh");
// String sPrepHydElemFP = "#! /bin/bash\n";
// sPrepHydElemFP += "wget \"" + hydElemPtr + "\" -O hydelem_file.tmp\n";
// sPrepHydElemFP += "flowpathline=`grep HYD_SYSTEM_FLOW_PATH_TYPE hydelem_file.tmp`\n";
// sPrepHydElemFP += "flowpathfile=`echo $flowpathline | cut -d\"'\" -f 2`\n";
// sPrepHydElemFP += "wget \"" + r2db + "/hydraulic-element-flow-paths/" + "$flowpathfile.xml\" -O hydelemflowpath_file.tmp\n";
// sPrepHydElemFP += "echo \"<Obj>\" > hydelemflowpath_file.xml\n";
// sPrepHydElemFP += "echo \"<Filename>hydraulic-element-flow-paths\\hydelemflowpath1</Filename>\" >> hydelemflowpath_file.xml\n";
// sPrepHydElemFP += "tail -n +2 hydelemflowpath_file.tmp >> hydelemflowpath_file.xml\n";
// FileUtils.writeStringToFile(preparehydElemFP, sPrepHydElemFP);
// preparehydElemFP.setExecutable(true);
// pcPrepareHydElemFP.working_dir = workingDir.getParent();
// pcPrepareHydElemFP.exe = ".//preparehydraulic-element.sh";
// pcPrepareHydElemFP.args = new String []{};
// pcPrepareHydElemFP.execute();
// stdout += "\n" + pcPrepareHydElemFP.stdout;
// stderr += "\n" + pcPrepareHydElemFP.stderr;
// return pcPrepareHydElemFP.exitValue;
// }
void prepareHydraulicElementFlowPathJ(String hydElemPtr, String r2db, File workingDir) throws IOException {
final String hydelemFile = "hydelem_file.tmp";
String hydelemflowpathFile = "";
BufferedReader br = null;
BufferedWriter bw = null;
try {
// first open hydElemPtr file
getFile(hydElemPtr, workingDir.getParent(), hydelemFile);
// look for HYD_SYSTEM_FLOW_PATH_TYPE
LOG.info("trying to open --- " + workingDir.getParent() + "/" + hydelemFile);
File hydElemFile = new File(workingDir.getParent() + "/" + hydelemFile);
FileInputStream fis = new FileInputStream(hydElemFile);
br = new BufferedReader(new InputStreamReader(fis));
LOG.info("opened --- " + workingDir.getParent() + "/" + hydelemFile);
String inLine;
while ((inLine = br.readLine()) != null) {
if (inLine.contains("HYD_SYSTEM_FLOW_PATH_TYPE")) {
// get the text between the single two quotes- this will be the flowpathfile we need to get
// <ObR><Name>HYD_SYSTEM_FLOW_PATH_TYPE</Name><Data>'0.3% grade channel'</Data><Type>HYD_ELEMENT_FLOW_PATH</Type></ObR></Obj>
hydelemflowpathFile = inLine.split("'")[1];
LOG.info("Read hydelemflowpathFile as =" + hydelemflowpathFile);
}
}
String hydElemFlowPathPtr = r2db + "/hydraulic-element-flow-paths/" + hydelemflowpathFile + ".xml";
LOG.info("HydelemflowpathPTR is =" + hydElemFlowPathPtr);
// call ingprepareFileJ to finish the work...
// next open the flowpathfile, and perform the standard formatting
// of adding the <Obj> tag and the <Filename> tag...
prepareFileJ(hydElemFlowPathPtr, new File(workingDir.getParent(), "hydelemflowpath_file.xml"), "", "hydraulic-element-flow-paths", "hydelemflowpath", false);
} catch (Exception ioe) {
LOG.info("IO Exception while preparing file=" + ioe.toString());
ioe.printStackTrace();
} finally {
if (br != null) {
br.close();
}
if (bw != null) {
bw.close();
}
}
}
int DetermineNumberOfFlowPaths(String hydElemPtr) {
hydElemPtr = hydElemPtr.substring(hydElemPtr.lastIndexOf("\\") + 1);
// This rediumentary algorithm for determining the number of flow paths for
// a hydaulic element system is based on conversation with Jack Carlson in late 03/2016
LOG.info("***********************************************************************************\n\n\n\nHYD ELEM PTR=" + hydElemPtr);
//if (hydElemPtr.substring(hydElemPtr.lastIndexOf("\")+1).startsWith("1"))
if (hydElemPtr.startsWith("1")) {
if (hydElemPtr.contains("middle")) {
return 2;
} else {
return 1;
}
}
// presently there are only 2 kinds of hydraulic element systems with 2 elements.
// ones with only "middle" in the name which should have 3 flow paths (all evenly distributed along the slope),
// and ones with "middle" and "bottom" in the name which should have 2 flow paths (one in middle, one at bottom).
//
// presently there is one hydraulic element with "along" instead of "middle" in the name
// "2 Water and Sediment Control Basins along RUSLE slope.xml" This should be treated the same as middle.
//
if (hydElemPtr.startsWith("2")) {
if (((hydElemPtr.contains("middle")) || (hydElemPtr.contains("along"))) && (!hydElemPtr.contains("bottom"))) {
return 3;
} else // These should have middle and bottom
{
return 2;
}
}
// presently there are only 2 kinds of hydraulic element systems with 3 elements.
// ones with only "middle" in the name which should have 4 flow paths (all evenly distributed along the slope),
// and ones with "middle" and "bottom" in the name which should have 3 flow paths (two in middle, one at bottom).
if (hydElemPtr.startsWith("3")) {
if ((hydElemPtr.contains("middle")) && (!hydElemPtr.contains("bottom"))) {
return 4;
} else // These should have middle and bottom
{
return 3;
}
}
// Should not get here.
return 0;
}
double[] DetermineFlowPathDistribution(String hydElemPtr) {
hydElemPtr = hydElemPtr.substring(hydElemPtr.lastIndexOf("\\") + 1);
// This rediumentary algorithm for determining the hyd elem flow path position relative to slope length for
// a hydaulic element system is based on conversation with Jack Carlson in late 03/2016
if (hydElemPtr.substring(hydElemPtr.lastIndexOf("\\") + 1).startsWith("1")) {
if (hydElemPtr.contains("middle")) {
return new double[]{.5, 1};
} else {
return new double[]{1};
}
}
// presently there are only 2 kinds of hydraulic element systems with 2 elements.
// ones with only "middle" in the name which should have 3 flow paths (all evenly distributed along the slope),
// and ones with "middle" and "bottom" in the name which should have 2 flow paths (one in middle, one at bottom).
//
// presently there is one hydraulic element with "along" instead of "middle" in the name
// "2 Water and Sediment Control Basins along RUSLE slope.xml" This should be treated the same as middle.
//
if (hydElemPtr.startsWith("2")) {
if (((hydElemPtr.contains("middle")) || (hydElemPtr.contains("along"))) && (!hydElemPtr.contains("bottom"))) {
return new double[]{.333, .666, 1};
} else // These should have middle and bottom
{
return new double[]{.5, 1};
}
}
// presently there are only 2 kinds of hydraulic element systems with 3 elements.
// ones with only "middle" in the name which should have 4 flow paths (all evenly distributed along the slope),
// and ones with "middle" and "bottom" in the name which should have 3 flow paths (two in middle, one at bottom).
if (hydElemPtr.startsWith("3")) {
if ((hydElemPtr.contains("middle")) && (!hydElemPtr.contains("bottom"))) {
return new double[]{.25, .5, .75, 1};
} else // These should have middle and bottom
{
return new double[]{.333, .666, 1};
}
}
// Should not get here...
return new double[]{0};
}
// Generic File Preparation routine
// Downloads, and prepares a file for use in Rusle2 Pyrome
//
// fileType: the type of file to process
// Valid fileType(s) include: "contour",
//
// fileDir: the dir of the filename specified in the <Filename> tag in the R2 XML file
// Valid fileDir(s) include: "countour-systems"
//
// idx: An index value, if there are multiple items of the same type for slope segments
// @Deprecated
// int prepareFile(String httpPtr, File outputFile, String fileType, String fileDir, String idx) throws IOException
// {
// ProcessComponent pcPrepare = new ProcessComponent();
// // Generate shell script to prepare --fileType-- file for pyrome
// LOG.info("***********************************************************************************the " + fileType + " file name=" + outputFile.getName());
// String tmpFilename = outputFile.getName().substring(0, outputFile.getName().length()-4) + ".tmp";
// LOG.info("***********************************************************************************the " + fileType + " tmp file name=" + tmpFilename);
// File prepare = new File(outputFile.getParent() + "/prepare" + fileType + ".sh");
// String prep = "#! /bin/bash\n" ;
// prep += "wget \"" + httpPtr + "\" -O " + tmpFilename + "\n";
// //prep += "echo \"<?xml version=\\\"1.0\\\"?>\" > " + workingDir.getName() + "\n";
// prep += "echo \"<Obj>\" >> " + outputFile.getName() + "\n";
// //prep += "echo \"<Type>" + fileType.toUpperCase() + "</Type>\" >> " + workingDir.getName() + "\n";
// prep += "echo \"<Filename>" + fileDir + "\\\\aaa" + idx + "</Filename>\" >> " + outputFile.getName() + "\n";
// //prep += "echo \"<Science>20030415</Science>\" >> " + workingDir.getName() + "\n";
// prep += "tail -n +2 " + tmpFilename + " >> " + outputFile.getName() + "\n";
// FileUtils.writeStringToFile(prepare, prep);
// prepare.setExecutable(true);
// LOG.info(fileType + "--HttpPtr=" + httpPtr);
// LOG.info("the pc-exe is=" + pcPrepare.exe);
// pcPrepare.working_dir = outputFile.getParent();
// pcPrepare.exe = "./prepare" + fileType + ".sh";
// pcPrepare.args = new String[]{};
// pcPrepare.execute();
// stdout += "\n" + pcPrepare.stdout;
// stderr += "\n" + pcPrepare.stderr;
// return pcPrepare.exitValue;
// }
void prepareFileJ(String httpPtr, File outputFile, String fileType, String fileDir, String basefilename, boolean prepareSoilForPyrome) throws IOException {
BufferedReader br = null;
BufferedWriter bw = null;
String fn = (((basefilename != null) & (basefilename.length() > 0)) ? "\\" + basefilename + "1" : "\\aaa");
try {
// Generate shell script to prepare --fileType-- file for pyrome
LOG.info("***********************************************************************************the " + fileType + " file name=" + outputFile.getName());
String tmpFilename = outputFile.getName().substring(0, outputFile.getName().length() - 4) + ".tmp";
LOG.info("***********************************************************************************the " + fileType + " tmp file name=" + tmpFilename);
getFile(httpPtr, outputFile.getParent(), tmpFilename);
FileOutputStream fos = new FileOutputStream(outputFile);
bw = new BufferedWriter(new OutputStreamWriter(fos));
File origFile = new File(outputFile.getParent() + "/" + tmpFilename);
FileInputStream fis = new FileInputStream(origFile);
br = new BufferedReader(new InputStreamReader(fis));
if ((fileType.contentEquals("CLIMATE")) || (fileType.contentEquals("SOIL")) || (prepareSoilForPyrome)) {
bw.write("<?xml version=\"1.0\"?>\n");
}
if (!fileType.contentEquals("SOIL")) {
bw.write("<Obj>\n");
if ((fileType != null) && (fileType.length() > 1)) {
bw.write("<Type>" + fileType + "</Type>\n");
}
// if an incrementing index is needed, replace "0" with counter variable
bw.write("<Filename>" + fileDir + fn + "</Filename>\n");
}
int i = 0;
String inputLine;
while ((inputLine = br.readLine()) != null) {
// skip the first line of the file, except for soil files which only have 1 line
if ((i > 0) || (fileType.contentEquals("SOIL") && (i == 0) && (inputLine.startsWith("<Obj>")))) {
bw.write(inputLine + "\n");
}
i++;
}
fis.close();
} catch (IOException ioe) {
LOG.info("IO Exception while preparing flie=" + outputFile.getName() + "---" + ioe.toString());
} finally {
if (br != null) {
br.close();
}
if (bw != null) {
bw.close();
}
}
}
// TO DO
// Consider moving to a helper class since this is a generally applicable function
private void getFile(String url, String destDir, String filename) {
FileOutputStream fos = null;
String escapedUrl = XMLUtils.escapeURL(url);
try {
LOG.info("*************************************\n\n\n\nTRYING TO GET FILE=" + url + "\n");
LOG.info("Where the URI is =");
LOG.info(escapedUrl + "\n\n\n\n\n*************************************************************");
HttpClient client = new DefaultHttpClient();
HttpGet httpget = new HttpGet(escapedUrl);
HttpResponse response = client.execute((HttpUriRequest) httpget);
byte[] soilData = IOUtils.toByteArray(response.getEntity().getContent());
fos = new FileOutputStream(new File(destDir + "/" + filename));
fos.write(soilData);
fos.close();
} catch (IOException ie) {
LOG.log(Level.SEVERE, "ERROR GETTING SOILS IFC FILE!:" + ie.toString());
throw new RuntimeException(ie);
} finally {
if (fos != null) {
try {
fos.close();
} catch (Exception fe) {
}
}
}
}
/**
* Gets results returned from RomeShell
*
* @param key
* @return
*/
protected String getResult(String key) {
String sPattern = "RomeFileGetAttrValue " + key;
//LOG.info("seeking='" + sPattern + "'");
// Check if attr is available via the GetAttrValue cmd and return if so
if (stdout.contains(sPattern)) {
int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
int end = stdout.toString().indexOf('\r', start);
//LOG.info("returning='" + stdout.substring(start, end).trim() + "'");
return stdout.substring(start, end).trim();
}
// Check if attr is available via the TestAttrValue cmd and return if so
sPattern = "RomeFileTestAttrValue " + key;
//LOG.info("seeking='" + sPattern + "'");
if (stdout.contains(sPattern)) {
int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
int end = stdout.toString().indexOf('\r', start);
//LOG.info("returning='" + stdout.substring(start, end).trim() + "'");
return stdout.substring(start, end).trim();
}
//String key2 = "\"#RD:MAN_PTR:#RD:MAN_IRRIG_SUB_OBJ_PTR:" + key.substring(1);
sPattern = ":" + removeFirstLastChar(key) + "\"";
//LOG.info("seeking='" + sPattern + "'");
if (stdout.contains(sPattern)) {
int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
int end = stdout.toString().indexOf('\r', start);
String output = stdout.substring(start, end).trim();
if ((output == null) || (output.length() <= 0)) {
output = "null";
}
//LOG.info("returning='" + output + "'");
return output;
}
// If attr is not available return null
LOG.info("returning='null'");
return "null";
}
/**
* Gets results returned from Pyrome
*
* @param key
* @return
*/
protected String getResultPyrome(String key) {
String sPattern = key + "=";
if (stdout.contains(key)) {
int start = stdout.indexOf(key) + key.length() + 1;
int end = stdout.toString().indexOf('\r', start);
//LOG.info("returning='" + stdout.substring(start, end).trim() + "'");
return stdout.substring(start, end).trim();
}
LOG.info("returning='null'");
return "null";
}
/**
* Gets results returned from Pyrome
*
* @param key
* @return
*/
protected String getResultPyromeArray(String key, Boolean bQuoted, Boolean bEscape, Boolean bIndexed) {
String result = "[";
String sPattern = "";
String text = "";
int idx = 0;
do {
sPattern = bIndexed ? (key + ":" + idx) : key;
System.out.println("Searching for key=" + sPattern);
text = getResultPyrome(sPattern);
System.out.println("Result of search=" + text);
if (!text.contentEquals("null")) {
if (idx > 0) {
result += ",";
}
if (bQuoted) {
result += "\"" + (bEscape ? StringEscapeUtils.escapeJava(text) : text) + "\"";
} else {
result += (bEscape ? StringEscapeUtils.escapeJava(text) : text);
}
idx++;
}
} while (!text.contentEquals("null"));
result += "]";
return result;
}
public boolean isUrlReachable(String targetUrl) throws IOException {
String r2db = Config.getString("r2.db", "http://oms-db.engr.colostate.edu/r2");
String testUrl = r2db + "/" + targetUrl;
testUrl = XMLUtils.escapeXMLFileURL(testUrl);
System.out.println("Testing URL=" + testUrl);
URL url = new URL(testUrl);
HttpURLConnection httpUrlConnection = (HttpURLConnection) url.openConnection();
//httpUrlConnection.setRequestMethod("HEAD");
httpUrlConnection.setRequestMethod("GET");
try {
int responseCode = httpUrlConnection.getResponseCode();
System.out.println("response code=" + responseCode);
return responseCode == HttpURLConnection.HTTP_OK;
} catch (UnknownHostException noInternetConnection) {
return false;
}
}
protected static String escapeJavaNoFwdSlash(String text) {
String javaEsc = StringEscapeUtils.escapeJava(text);
return javaEsc.replace("\\/", "/");
}
public static String removeFirstLastChar(String text) {
return text.substring(1, text.length() - 1);
}
}