R2Run.java [src/java/m/rusle2] Revision: aa66a175401db0f372bde6f7064de068e88d6e9a  Date: Thu Apr 07 17:26:06 MDT 2016
/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package m.rusle2;

import csip.Config;
import java.io.File;
import java.io.IOException;
import java.util.logging.Logger;
import oms3.util.ProcessComponent;
import csip.utils.Binaries;
import csip.utils.Services;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringEscapeUtils;

/**
 *
 * @author od
 */
class R2Run {

    public static final Logger LOG = Logger.getLogger("R2Run");
    String stdout;
    String stderr;
    ProcessComponent pc = new ProcessComponent();
    
    
    /**
     * Execute the r2 script.
     *
     * @param r2_rsh
     * @return
     * @throws IOException
     */
    int execute(File r2_rsh) throws IOException {
        LOG.info("Execute method");
        String binDir = Config.getString("m.bin.dir", "/tmp/csip/bin");

        File romeshell = Binaries.unpackResource("/bin/win-x86/RomeShell.exe", new File(binDir));
        Binaries.unpackResource("/bin/win-x86/RomeDLL.dll", new File(binDir));

        pc.exe = Config.getString("wine.path", "/usr/bin/wine");
        pc.working_dir = r2_rsh.getParent();
        pc.args = new String[]{romeshell.toString(), r2_rsh.toString()};
        LOG.info("Executing");
        pc.execute();
        LOG.info("executed");
        stdout += "\n" + pc.stdout;
        stderr += "\n" + pc.stderr;
        LOG.info("stdout: "+ stdout);
        LOG.info(("stderr: "+ stderr));
        //LOG.info("rusle2 model run stdout=" + stdout);
        LOG.info("exit val: "+pc.exitValue);
        return pc.exitValue;
    }


    /**
     * Execute the rusle2 pyrome python script
     * 
     * We assume that Python3.4 is installed under /home/ubuntu/.wine/drive_c/Python34
     * and that wine's working dir is /home/ubuntu/.wine
     * The ownership of wine's working directory must correspond with the user running tomcat
     *
     * @param r2_rsh
     * @return
     * @throws IOException
     */
    int executePyrome(File r2_rsh) throws IOException {
        LOG.info("Execute method");
        String binDir = Config.getString("m.bin.dir", "/tmp/csip/bin");

        //File romeshell = Binaries.unpackResource("/bin/win-x86/RomeShell.exe", new File(binDir));
        Binaries.unpackResource("/bin/win-x86/RomeDLL.dll", new File(binDir));
        Binaries.unpackResource("/bin/win-x86/pyrome.py", new File(binDir));
        Binaries.unpackResource("/bin/win-x86/_pyrome.pyd", new File(binDir));  
        File runpyrome = new File(r2_rsh.getParent() + "/runrusle2.sh");
        String runrusle2 = "winetricks vd=off ; wine /home/ubuntu/.wine/drive_c/Python34/python.exe rusle2csip.py\n"; 
        FileUtils.writeStringToFile(runpyrome, runrusle2);
        runpyrome.setExecutable(true);
        pc.exe = "./runrusle2.sh";
        pc.working_dir = r2_rsh.getParent();
        pc.args = new String[]{};
        LOG.info("Executing pyrome rusle2");
        pc.execute();
        LOG.info("executed pyrome rusle2");
        stdout += "\n" + pc.stdout;
        stderr += "\n" + pc.stderr;
        LOG.info("stdout: "+ stdout);
        LOG.info(("stderr: "+ stderr));
        //LOG.info("rusle2 model run stdout=" + stdout);
        LOG.info("exit val: "+pc.exitValue);
        return pc.exitValue;
    }
    
    // to do
    // prepare functions need to be rewritten to be Java only (no shelling out to linux!)
    
    int prepareSoilsFile(String soilsHttpPtr, File workingDir, boolean bPyrome, String idx) throws IOException {
        // Generate shell script to prepare soils file for romeshell or pyrome
        ProcessComponent pcPrepareSoils = new ProcessComponent();        
        File preparesoils = new File(workingDir.getParent() + "/preparesoils" + idx + ".sh");
        String prepsoils = "wget \"" + soilsHttpPtr + "\" -O soils_file" + idx + ".tmp\n"; 
        if (bPyrome) 
        {
            prepsoils += "echo \"<?xml version=\"\\\"1.0\\\"\"?>\" > soils_file" + idx + ".xml\n";
        }
        prepsoils += "numlines=`cat soils_file" + idx + ".tmp | wc -l`\n";
        prepsoils += "if [ $numlines -eq 1 ]\n";
        prepsoils += "then\n";
        prepsoils += "  tail -n +1 soils_file" + idx + ".tmp >> soils_file" + idx + ".xml\n";
        prepsoils += "else\n";
        prepsoils += "  tail -n +2 soils_file" + idx + ".tmp >> soils_file" + idx + ".xml\n";
        prepsoils += "fi\n";
        FileUtils.writeStringToFile(preparesoils, prepsoils);
        preparesoils.setExecutable(true);
        LOG.info("soilsHttpPtr=" + soilsHttpPtr);
        LOG.info("the pc-exe is=" + pcPrepareSoils.exe);
        pcPrepareSoils.working_dir = workingDir.getParent();
        pcPrepareSoils.exe = "./preparesoils" + idx + ".sh";
        pcPrepareSoils.args = new String[]{};
        pcPrepareSoils.execute();
        stdout += "\n" + pcPrepareSoils.stdout;
        stderr += "\n" + pcPrepareSoils.stderr;
        return pcPrepareSoils.exitValue;
    }
    
    int prepareClimateFile(String climateHttpPtr, File workingDir) throws IOException
    {
        ProcessComponent pcPrepareCli = new ProcessComponent();
        // Generate shell script to prepare climate file for pyrome
        File preparecli = new File(workingDir.getParent() + "/preparecli.sh");
        String prepcli = "wget \"" + climateHttpPtr + "\" -O cli_file0.tmp\n"; 
        prepcli += "echo \"<?xml version=\\\"1.0\\\"?>\" > cli_file0.xml\n";
        prepcli += "echo \"<Obj>\" >> cli_file0.xml\n";
        prepcli += "echo \"<Type>CLIMATE</Type>\" >> cli_file0.xml\n";        
        prepcli += "echo \"<Filename>climates\\\\\\\\aaa</Filename>\" >> cli_file0.xml\n";
        prepcli += "tail -n +2 cli_file0.tmp >> cli_file0.xml\n";
        FileUtils.writeStringToFile(preparecli, prepcli);
        preparecli.setExecutable(true);
        LOG.info("climateHttpPtr=" + climateHttpPtr);
        LOG.info("the pc-exe is=" + pcPrepareCli.exe);
        pcPrepareCli.working_dir = workingDir.getParent();
        pcPrepareCli.exe = "./preparecli.sh";
        pcPrepareCli.args = new String[]{};
        pcPrepareCli.execute();
        stdout += "\n" + pcPrepareCli.stdout;
        stderr += "\n" + pcPrepareCli.stderr;
        return pcPrepareCli.exitValue;
    }
    
    // Must find the hydraulic element flow path element of a hydraulic element system
    // download this file, and modify the header, so it can be fed to python
    int prepareHydraulicElementFlowPath(String hydElemPtr, String r2db, File workingDir) throws IOException
    {
        ProcessComponent pcPrepareHydElemFP = new ProcessComponent();
        File preparehydElemFP = new File(workingDir.getParent() + "/preparehydraulic-element.sh");
        String sPrepHydElemFP = "#! /bin/bash\n";
        sPrepHydElemFP += "wget \"" + hydElemPtr + "\" -O hydelem_file.tmp\n";
        sPrepHydElemFP += "flowpathline=`grep HYD_SYSTEM_FLOW_PATH_TYPE hydelem_file.tmp`\n";
        sPrepHydElemFP += "flowpathfile=`echo $flowpathline | cut -d\"'\" -f 2`\n";
        sPrepHydElemFP += "wget \"" + r2db + "/hydraulic-element-flow-paths/" + "$flowpathfile.xml\" -O hydelemflowpath_file.tmp\n";
        sPrepHydElemFP += "echo \"<Obj>\" > hydelemflowpath_file.xml\n";
        sPrepHydElemFP += "echo \"<Filename>hydraulic-element-flow-paths\\hydelemflowpath1</Filename>\" >> hydelemflowpath_file.xml\n";
        sPrepHydElemFP += "tail -n +2 hydelemflowpath_file.tmp >> hydelemflowpath_file.xml\n";
        FileUtils.writeStringToFile(preparehydElemFP, sPrepHydElemFP);
        preparehydElemFP.setExecutable(true);
        pcPrepareHydElemFP.working_dir = workingDir.getParent();
        pcPrepareHydElemFP.exe = ".//preparehydraulic-element.sh";
        pcPrepareHydElemFP.args = new String []{};
        pcPrepareHydElemFP.execute();
        stdout += "\n" + pcPrepareHydElemFP.stdout;
        stderr += "\n" + pcPrepareHydElemFP.stderr;
        return pcPrepareHydElemFP.exitValue;
    }
    
    int DetermineNumberOfFlowPaths(String hydElemPtr) 
    {
        hydElemPtr = hydElemPtr.substring(hydElemPtr.lastIndexOf("\\")+1);

        // This rediumentary algorithm for determining the number of flow paths for
        // a hydaulic element system is based on conversation with Jack Carlson in late 03/2016
        LOG.info("***********************************************************************************\n\n\n\nHYD ELEM PTR=" + hydElemPtr);        
        //if (hydElemPtr.substring(hydElemPtr.lastIndexOf("\")+1).startsWith("1"))
        if (hydElemPtr.startsWith("1"))
        {
            if (hydElemPtr.contains("middle"))
                return 2;
            else
                return 1;
        }
        
        // presently there are only 2 kinds of hydraulic element systems with 2 elements.  
        // ones with only "middle" in the name which should have 3 flow paths (all evenly distributed along the slope),
        // and ones with "middle" and "bottom" in the name which should have 2 flow paths (one in middle, one at bottom).
        //
        // presently there is one hydraulic element with "along" instead of "middle" in the name
        // "2 Water and Sediment Control Basins along RUSLE slope.xml"  This should be treated the same as middle.
        // 
        if (hydElemPtr.startsWith("2"))
        {
            if ( ((hydElemPtr.contains("middle")) || (hydElemPtr.contains("along"))) && (!hydElemPtr.contains("bottom")))
                return 3;
            else
                // These should have middle and bottom
                return 2;
        }

        // presently there are only 2 kinds of hydraulic element systems with 3 elements.  
        // ones with only "middle" in the name which should have 4 flow paths (all evenly distributed along the slope),
        // and ones with "middle" and "bottom" in the name which should have 3 flow paths (two in middle, one at bottom).
        if (hydElemPtr.startsWith("3"))
        {
            if ((hydElemPtr.contains("middle")) && (!hydElemPtr.contains("bottom")))
                return 4;
            else
                // These should have middle and bottom
                return 3;            
        }
        // Should not get here.
        return 0;
    }
    
    double[] DetermineFlowPathDistribution(String hydElemPtr)
    {
        hydElemPtr = hydElemPtr.substring(hydElemPtr.lastIndexOf("\\")+1);
        // This rediumentary algorithm for determining the hyd elem flow path position relative to slope length for
        // a hydaulic element system is based on conversation with Jack Carlson in late 03/2016
        
        if (hydElemPtr.substring( hydElemPtr.lastIndexOf("\\")+1 ).startsWith("1"))
        {
            if (hydElemPtr.contains("middle"))
                return new double[]{.5, 1};
            else
                return new double[]{1};
        }
        
        // presently there are only 2 kinds of hydraulic element systems with 2 elements.  
        // ones with only "middle" in the name which should have 3 flow paths (all evenly distributed along the slope),
        // and ones with "middle" and "bottom" in the name which should have 2 flow paths (one in middle, one at bottom).
        //
        // presently there is one hydraulic element with "along" instead of "middle" in the name
        // "2 Water and Sediment Control Basins along RUSLE slope.xml"  This should be treated the same as middle.
        // 
        if (hydElemPtr.startsWith("2"))
        {
            if ( ((hydElemPtr.contains("middle")) || (hydElemPtr.contains("along"))) && (!hydElemPtr.contains("bottom")))
                return new double[]{.333,.666,1};
            else
                // These should have middle and bottom
                return new double[]{.5, 1};
        }

        // presently there are only 2 kinds of hydraulic element systems with 3 elements.  
        // ones with only "middle" in the name which should have 4 flow paths (all evenly distributed along the slope),
        // and ones with "middle" and "bottom" in the name which should have 3 flow paths (two in middle, one at bottom).
        if (hydElemPtr.startsWith("3"))
        {
            if ((hydElemPtr.contains("middle")) && (!hydElemPtr.contains("bottom")))
                return new double[]{.25,.5,.75,1};
            else
                // These should have middle and bottom
                return new double[]{.333,.666,1};            
        }
        // Should not get here...
        return new double[]{0};        
    }
    
//    int prepareContourFile(String contourHttpPtr, File workingDir) throws IOException
//    {
//        ProcessComponent pcPrepareContour = new ProcessComponent();
//        // Generate shell script to prepare climate file for pyrome
//        LOG.info("***********************************************************************************the contour file name=" + workingDir.getName());
//        String tmpFilename = workingDir.getName().substring(0, workingDir.getName().length()-4);
//        LOG.info("***********************************************************************************the contour tmp file name=" + tmpFilename);
//        File preparecontour = new File(workingDir.getParent() + "/preparecontour.sh");
//        String prepcontour = "wget \"" + contourHttpPtr + "\" -O " + tmpFilename + "\n"; 
//        prepcontour += "echo \"<?xml version=\\\"1.0\\\"?>\" > " + workingDir.getName() + "\n";
//        prepcontour += "echo \"<Obj>\" >> " + workingDir.getName() + "\n";
//        prepcontour += "echo \"<Type>CONTOUR</Type>\" >> " + workingDir.getName() + "\n";        
//        prepcontour += "echo \"<Filename>contour-systems\\\\\\\\aaa</Filename>\" >> " + workingDir.getName() + "\n";
//        prepcontour += "tail -n +2 contour_file0.tmp >> " + workingDir.getName() + "\n";
//        FileUtils.writeStringToFile(preparecontour, prepcontour);
//        preparecontour.setExecutable(true);
//        LOG.info("contourHttpPtr=" + contourHttpPtr);
//        LOG.info("the pc-exe is=" + pcPrepareContour.exe);
//        pcPrepareContour.working_dir = workingDir.getParent();
//        pcPrepareContour.exe = "./preparecontour.sh";
//        pcPrepareContour.args = new String[]{};
//        pcPrepareContour.execute();
//        stdout += "\n" + pcPrepareContour.stdout;
//        stderr += "\n" + pcPrepareContour.stderr;
//        return pcPrepareContour.exitValue;
//    }

    // Generic File Preparation routine
    // Downloads, and prepares a file for use in Rusle2 Pyrome
    //
    // fileType: the type of file to process
    // Valid fileType(s) include: "contour",  
    //
    // fileDir: the dir of the filename specified in the <Filename> tag in the R2 XML file
    // Valid fileDir(s) include: "countour-systems"
    //
    // idx: An index value, if there are multiple items of the same type for slope segments
    int prepareFile(String httpPtr, File workingDir, String fileType, String fileDir, String idx) throws IOException
    {
        ProcessComponent pcPrepare = new ProcessComponent();
        // Generate shell script to prepare --fileType-- file for pyrome
        LOG.info("***********************************************************************************the " + fileType + " file name=" + workingDir.getName());
        String tmpFilename = workingDir.getName().substring(0, workingDir.getName().length()-4) + ".tmp";
        LOG.info("***********************************************************************************the " + fileType + " tmp file name=" + tmpFilename);
        File prepare = new File(workingDir.getParent() + "/prepare" + fileType + ".sh");
        String prep = "#! /bin/bash\n" ;
        prep += "wget \"" + httpPtr + "\" -O " + tmpFilename + "\n"; 
        //prep += "echo \"<?xml version=\\\"1.0\\\"?>\" > " + workingDir.getName() + "\n";
        prep += "echo \"<Obj>\" >> " + workingDir.getName() + "\n";
        //prep += "echo \"<Type>" + fileType.toUpperCase() + "</Type>\" >> " + workingDir.getName() + "\n";        
        prep += "echo \"<Filename>" + fileDir + "\\\\aaa" + idx + "</Filename>\" >> " + workingDir.getName() + "\n";
        //prep += "echo \"<Science>20030415</Science>\" >> " + workingDir.getName() + "\n";
        prep += "tail -n +2 " + tmpFilename + " >> " + workingDir.getName() + "\n";
        FileUtils.writeStringToFile(prepare, prep);
        prepare.setExecutable(true);
        LOG.info(fileType + "--HttpPtr=" + httpPtr);
        LOG.info("the pc-exe is=" + pcPrepare.exe);
        pcPrepare.working_dir = workingDir.getParent();
        pcPrepare.exe = "./prepare" + fileType + ".sh";
        pcPrepare.args = new String[]{};
        pcPrepare.execute();
        stdout += "\n" + pcPrepare.stdout;
        stderr += "\n" + pcPrepare.stderr;
        return pcPrepare.exitValue;
    }
    

    /**
     * Gets results returned from RomeShell
     * @param key
     * @return
     */
    protected String getResult(String key) {
        String sPattern = "RomeFileGetAttrValue " + key;
        
        //LOG.info("seeking='" + sPattern + "'");
        // Check if attr is available via the GetAttrValue cmd and return if so
        if (stdout.contains(sPattern))
        {
            int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
            int end = stdout.toString().indexOf('\r', start); 
            //LOG.info("returning='" + stdout.substring(start, end).trim() + "'");
            return stdout.substring(start, end).trim();
        }
        
        // Check if attr is available via the TestAttrValue cmd and return if so
        sPattern = "RomeFileTestAttrValue " + key;
        //LOG.info("seeking='" + sPattern + "'");        
        if (stdout.contains(sPattern))
        {
            int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
            int end = stdout.toString().indexOf('\r', start);   
            //LOG.info("returning='" + stdout.substring(start, end).trim() + "'");            
            return stdout.substring(start, end).trim();
        }

        //String key2 = "\"#RD:MAN_PTR:#RD:MAN_IRRIG_SUB_OBJ_PTR:" + key.substring(1);
        sPattern = ":" + Services.removeFirstLastChar(key) + "\"";
        //LOG.info("seeking='" +  sPattern + "'");        
        if (stdout.contains(sPattern))
        {
            int start = stdout.indexOf(sPattern) + sPattern.length() + 2;
            int end = stdout.toString().indexOf('\r', start); 
            String output = stdout.substring(start, end).trim();
            if ((output == null) || (output.length() <= 0))
                output = "null";
            //LOG.info("returning='" + output + "'");            
            return output;
        }

        // If attr is not available return null
        LOG.info("returning='null'");
        return "null";
    }
    
    /**
     * Gets results returned from Pyrome
     * @param key
     * @return
     */
    protected String getResultPyrome(String key) {
        String sPattern = key + "=";
        
        if (stdout.contains(key))
        {
            int start = stdout.indexOf(key) + key.length() + 1;
            int end = stdout.toString().indexOf('\r', start); 
            //LOG.info("returning='" + stdout.substring(start, end).trim() + "'");
            return stdout.substring(start, end).trim();
        }
        
        LOG.info("returning='null'");
        return "null";
    }

    /**
     * Gets results returned from Pyrome
     * @param key
     * @return
     */
    protected String getResultPyromeArray(String key, Boolean bQuoted, Boolean bEscape) {
        String result = "[";
        String sPattern = "";
        String text = "";
        int idx = 0;
        do
        {
            sPattern = key + ":" + idx;
            text = getResultPyrome(sPattern);
            if (!text.contentEquals("null"))
            {
                if (idx > 0)
                    result += ",";
                if (bQuoted)
                    result += "\"" + (bEscape ? StringEscapeUtils.escapeJava(text) : text) + "\"";
                else
                    result += (bEscape ? StringEscapeUtils.escapeJava(text) : text);
                idx ++;
            }
        } while(!text.contentEquals("null"));
        
        result += "]";
        return result;
    }
    
    public boolean isUrlReachable(String targetUrl) throws IOException
    {
        String r2db = Config.getString("r2.db", "http://oms-db.engr.colostate.edu/r2");
        String testUrl = r2db + "/" + targetUrl;
        testUrl = XMLUtils.escapeURL(testUrl);
        System.out.println("Testing URL=" + testUrl);
        URL url = new URL(testUrl);
        
        HttpURLConnection httpUrlConnection = (HttpURLConnection) url.openConnection();
        //httpUrlConnection.setRequestMethod("HEAD");
        httpUrlConnection.setRequestMethod("GET");
        try
        {
            int responseCode = httpUrlConnection.getResponseCode();
            System.out.println("response code=" + responseCode);
            return responseCode == HttpURLConnection.HTTP_OK;
        } 
        catch (UnknownHostException noInternetConnection)
        {
            return false;
        }
    }
    
}