V1_0.java [src/java/m/crp/assessmenttool] Revision: default  Date:
/*
 * $Id$
 *
 * This file is part of the Cloud Services Integration Platform (CSIP),
 * a Model-as-a-Service framework, API, and application suite.
 *
 * 2012-2019, OMSLab, Colorado State University.
 *
 * OMSLab licenses this file to you under the MIT license.
 * See the LICENSE file in the project root for more information.
 */
package m.crp.assessmenttool;

import crp.utils.DBResources;
import static crp.utils.DBResources.LOCAL_SQLSERVER;
import crp.utils.DEMSteepness;
import crp.utils.Region;
import crp.utils.ServiceCall;
import crp.utils.SoilResult;
import crp.utils.WEPP;
import crp.utils.WEPS;
import crp.utils.WEPSFallowRotation;
import crp.utils.WWESoilParams;
import csip.Config;
import csip.ModelDataService;
import csip.PayloadParameter;
import csip.PayloadResults;
import csip.ServiceException;
import csip.annotations.Description;
import csip.annotations.Name;
import csip.annotations.Polling;
import csip.annotations.Resource;
import csip.utils.JSONUtils;
import data.interpretors.SlopeSteepness;
import data.interpretors.SlopeSteepness.SlopeData;
import gisobjects.GISObject;
import gisobjects.GISObjectException;
import gisobjects.GISObjectFactory;
import gisobjects.db.GISEngine;
import gisobjects.db.GISEngineFactory;
import gisobjects.vector.GIS_FeatureCollection;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ws.rs.Path;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import soils.utils.EvalResult;

/**
 *
 * @author <a href="mailto:shaun.case@colostate.edu">Shaun Case</a>
 */
@Name("CRP Assessment")
@Description("This service consumes a JSON request containing a request identifier and CRP Offer geometry and returns sheet/rill water and wind erosion rates for a fallow management for up to the three most dominant soil components in a CRP Offer area, plus a weighted average for each erosion rate.")
@Path("m/crpassessment/1.0")
@Polling(first = 2000, next = 1000)
@Resource(from = DBResources.class)

public class V1_0 extends ModelDataService {

  public static final String SCENARIO_ID = "scenario_id";
  public static final String SCENARIO_GEOMETRY = "scenario_geometry";
  public static final String USE_DEM_SLOPE_SERVICE = "use_dem_slope";
  public static final String SHOW_DEBUG_OUTPUT = "show_debug_output";
  public static final String COMPARE_SLOPE_METHODS = "compare_slope_methods";
  protected static final int SLEEP_TIMER = 500; //milliseconds to sleep while polling async model calls
  protected static final int INITIAL_POLL_WAIT_TIMER = 1000; //milliseconds to sleep while polling async model calls
  protected static final int GET_CALL_WAIT = 100; // milliseconds to sleep between internal get requests in loop of testing status of all calls at once.

  protected JSONObject scenario_geometry;
  protected String scenario_id;
  protected GISObject aoa_geometry;
  protected JSONObject dem_featureCollection;
  protected GISEngine gEngine;

  protected String regionURI = "http://csip.engr.colostate.edu:8083/csip-weps/m/region/2.0";
  protected String soilsURI = "http://csip.engr.colostate.edu:8092/csip-soils/d/wwesoilparams/2.1";
  protected String weppURI = "http://csip.engr.colostate.edu:8083/csip-wepp/m/wepp/3.1";
  protected String wepsURI = "http://csip.engr.colostate.edu:8083/csip-weps/m/weps/5.2";
  protected String demSteepnessURI = "http://csip.engr.colostate.edu:8087/csip-watershed/m/average_slope/3.0";

  protected String outputFile;
  protected boolean streamOutputFile = false;

  protected double regionLength, regionWidth, regionOrientation;
  protected ArrayList<SoilResult> topThreeComponents;
  protected ArrayList<SoilResult> alternateSlopeTopThreeComponents;
  protected double latitude, longitude;
  protected JSONObject rotation;
  protected double waterErosionRate = 0.0;
  protected double windErosionRate = 0.0;
  protected SlopeSteepness slopes;
  protected boolean useDemSlope = true;
  protected boolean debugOutput = false;
  protected boolean compareSlopes = true;

  protected ArrayList<ASYNCCall> weppCalls = new ArrayList<>();
  protected ArrayList<ASYNCCall> wepsCalls = new ArrayList<>();

  @Override
  protected void preProcess() throws Exception {
    PayloadParameter params = parameter();

    soilsURI = Config.getString("crp.soils", soilsURI);
    regionURI = Config.getString("crp.region", regionURI);
    weppURI = Config.getString("crp.wepp", weppURI);
    wepsURI = Config.getString("crp.weps", wepsURI);
    demSteepnessURI = Config.getString("crp.dem.steepness", demSteepnessURI);

    //  Keep in this order.
    compareSlopes = params.getBoolean(COMPARE_SLOPE_METHODS, false);
    debugOutput = params.getBoolean(SHOW_DEBUG_OUTPUT, false) | compareSlopes;

    if (params.has(SCENARIO_ID)) {
      scenario_id = params.getString(SCENARIO_ID);
    } else {
      throw new ServiceException("A required input 'scenario_id' was missing.  Please specify a scenario_id value.");
    }

    if (params.has(USE_DEM_SLOPE_SERVICE)) {
      useDemSlope = params.getBoolean(USE_DEM_SLOPE_SERVICE);
    }

    if (params.has(SCENARIO_GEOMETRY)) {
      scenario_geometry = params.getParamJSON(SCENARIO_GEOMETRY);

      try (Connection connection = resources().getJDBC(LOCAL_SQLSERVER);) {
        gEngine = GISEngineFactory.createGISEngine(connection);
        aoa_geometry = GISObjectFactory.createGISObject(scenario_geometry, gEngine);

        if (aoa_geometry.getType() == GISObject.GISObjectType.featurecollection) {  //May it also be a GeometryCollection??
          //  This is okay, just need to step through each feature, and also need 
          // to submit the feature collection to watershed/m/averageslope (DEM service), making 
          // sure that each has a field in properties with which to index it.
          validateFeatureCollection((GIS_FeatureCollection) aoa_geometry);
          dem_featureCollection = scenario_geometry;
        } else {
          //  If this is a polygon/multipolygon object, need to convert 
          // it into a featurecollection for the averageslope (DEM service)
          if ((aoa_geometry.getType() == GISObject.GISObjectType.polygon
              || aoa_geometry.getType() == GISObject.GISObjectType.multipolygon)) {

            dem_featureCollection = convertPolyToFeatureCollection(aoa_geometry);
          } else {
            throw new ServiceException("The input geometry must be a Polygon, Multipolygon, or FeatureCollection.");
          }

        }

        //  Shortcut to getting the centroid of a shape...Built in to the GISObject source for getLat/Lon if shape is not a point.
        latitude = aoa_geometry.getLatitude();
        longitude = aoa_geometry.getLongitude();
      }

    } else {
      throw new ServiceException("A required input " + SCENARIO_GEOMETRY + " was missing.  Please specify a scenario_id value.");
    }

  }

  @Override
  protected void doProcess() throws Exception {

    //Get all soils information for this AoI.
    WWESoilParams soilsCall = new WWESoilParams(metainfo().toString(), scenario_geometry, soilsURI);
    soilsCall.call();
    topThreeComponents = soilsCall.getTopThree();

    //  Get region data
    Region regionCall = new Region(metainfo().toString(), scenario_geometry, regionURI);
    regionCall.call();
    regionLength = regionCall.getLength();
    regionWidth = regionCall.getWidth();
    regionOrientation = regionCall.getOrientation();

    if (useDemSlope || compareSlopes) {
      getDEMSlopes(topThreeComponents);
    }

    if (compareSlopes) {
      alternateSlopeTopThreeComponents = new ArrayList<>();
      for (SoilResult soilResult : topThreeComponents) {
        SoilResult tSoilResult = new SoilResult(soilResult);
        alternateSlopeTopThreeComponents.add(tSoilResult);
      }
    }

    //TODO:  Currently rotations are the default fallow operation...final version of this service requires user to provide rotation.
    double totalArea = 0.0;
    for (SoilResult soilResult : topThreeComponents) {
      soilResult.usedDEM = useDemSlope;
      WEPP weppCall = new WEPP(metainfo().toString(), latitude, longitude, soilResult.cokey,
          soilResult.length, ((!useDemSlope) ? soilResult.slope_r : (soilResult.slopeDEM * 100.0)), getRotation((int) soilResult.length), weppURI);
      weppCall.call();

      weppCalls.add(new ASYNCCall(soilResult, weppCall));

      //soilResult.waterErosion = weppCall.waterErosion();
      WEPS wepsCall = new WEPS(metainfo().toString(), latitude, longitude, 200,
          200, regionOrientation, soilResult.cokey, getRotation(0), wepsURI);
      wepsCall.call();
      wepsCalls.add(new ASYNCCall(soilResult, wepsCall));

      //soilResult.windErosion = wepsCall.windErosion();
      totalArea += soilResult.area;
    }

    if (compareSlopes) {
      for (SoilResult soilResult : alternateSlopeTopThreeComponents) {
        soilResult.usedDEM = !useDemSlope;
        WEPP weppCall = new WEPP(metainfo().toString(), latitude, longitude, soilResult.cokey,
            soilResult.length, (useDemSlope ? soilResult.slope_r : (soilResult.slopeDEM * 100.0)), getRotation((int) soilResult.length), weppURI);
        weppCall.call();

        weppCalls.add(new ASYNCCall(soilResult, weppCall));
      }
    }

    if (!wepsCalls.isEmpty() && !weppCalls.isEmpty()) {
      boolean doneWithCalls = false;
      int count = weppCalls.size() + wepsCalls.size();

      //  Probably should add an overall timeout value for this loop so that it 
      // doesn't runn forever in really odd edge cases.
      boolean initialPollWaited = false;
      while (!doneWithCalls) {
        int waitedMills = 0;

        try {
          //Wait an initial period before beginning polling all the model results.
          if (!initialPollWaited) {
            Thread.sleep(INITIAL_POLL_WAIT_TIMER);
            initialPollWaited = true;
          }

          for (ASYNCCall weppCall : weppCalls) {
            if (!weppCall.callFinished) {
              if (weppCall.serviceCall.isFinished() || weppCall.serviceCall.asyncDone()) {
                weppCall.soilResult.waterErosion = ((WEPP) weppCall.serviceCall).waterErosion();
                weppCall.callFinished = true;
                count--;
              }
              Thread.sleep(GET_CALL_WAIT);
              waitedMills += GET_CALL_WAIT;
            }
          }

          for (ASYNCCall wepsCall : wepsCalls) {
            if (!wepsCall.callFinished) {
              if (wepsCall.serviceCall.isFinished() || wepsCall.serviceCall.asyncDone()) {
                wepsCall.soilResult.windErosion = ((WEPS) wepsCall.serviceCall).windErosion();
                wepsCall.callFinished = true;
                count--;
              }
              Thread.sleep(GET_CALL_WAIT);
              waitedMills += GET_CALL_WAIT;
            }
          }

          if (waitedMills < SLEEP_TIMER) {
            Thread.sleep(SLEEP_TIMER - waitedMills);
          }
        } catch (InterruptedException ex) {
          if (count <= 0) {
            doneWithCalls = true;
          }
          for (ASYNCCall wepsCall : wepsCalls) {
            if (!wepsCall.serviceCall.isFinished()) {
              wepsCall.serviceCall.cancel();
            }
          }
          for (ASYNCCall weppCall : weppCalls) {
            if (!weppCall.serviceCall.isFinished()) {
              weppCall.serviceCall.cancel();
            }
          }
          //  Allow the thread to be killed...just in case.
          break;
        }
        if (count <= 0) {
          doneWithCalls = true;
        }
      }

      if (!doneWithCalls) {
        throw new ServiceException("Could not gather all of the results of the models running. " + count + " models' results were unaccounted for.");
      }
    } else {
      throw new ServiceException("No WEPP/WEPS model runs were created.  Cannot proceed.");
    }

    for (SoilResult soilResult : topThreeComponents) {
      if (Double.isNaN(soilResult.waterErosion) || Double.isNaN(soilResult.windErosion)) {
        throw new ServiceException("Not a number value encountered while trying to average wind or water erosion values.");
      }
      waterErosionRate += soilResult.waterErosion * (soilResult.area / totalArea);
      windErosionRate += soilResult.windErosion * (soilResult.area / totalArea);
    }
  }

  @Override
  protected void postProcess() throws Exception {
    PayloadResults results = results();
    JSONArray soilsResult = new JSONArray();

    for (SoilResult soilResult : topThreeComponents) {
      JSONArray soilData = new JSONArray();
      if (debugOutput) {
        soilData.put(JSONUtils.data("mukey", soilResult.mukey));
        soilData.put(JSONUtils.data("cokey", soilResult.cokey));
        soilData.put(JSONUtils.data("soilName", soilResult.soilName));
        soilData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
        soilData.put(JSONUtils.data("slope_r", soilResult.slope_r));
        soilData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100.0 : "NONE")));
        soilData.put(JSONUtils.data("length", soilResult.length));
        soilData.put(JSONUtils.data("soilName", soilResult.soilName));
        soilData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
      }

      soilData.put(JSONUtils.data("soilPtr", soilResult.cokey));
      soilData.put(JSONUtils.data("soilName", soilResult.soilName));
      soilData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
      soilData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
      soilData.put(JSONUtils.data("WaterSoilLoss", soilResult.waterErosion, "Average Annual Soil Loss by Water", "ton/ac/yr"));
      soilData.put(JSONUtils.data("WindSoilLoss", soilResult.windErosion, "Average Annual Soil Loss by Wind", "ton/ac/yr"));

      soilsResult.put(soilData);
    }
    results.put("top_3_soils", soilsResult);
    results.put("WtAvgWaterSoilLoss", waterErosionRate, "Weighted Average Annual Soil Loss by Water", "ton/ac/yr");
    results.put("WtAvgWindSoilLoss", windErosionRate, "Weighted Average Annual Soil Loss by Wind", "ton/ac/yr");
    results.put("WtAvgTotalSoilLoss", waterErosionRate + windErosionRate, "Weighted Average Annual Soil Loss by Water and Wind", "ton/ac/yr");

    if (debugOutput) {
      JSONArray soilsDebugResults = new JSONArray();
      JSONArray subCalls = new JSONArray();

      for (ASYNCCall aCall : weppCalls) {
        JSONArray allModelData = new JSONArray();
        SoilResult soilResult = aCall.soilResult;

        allModelData.put(JSONUtils.data("mukey", soilResult.mukey));
        allModelData.put(JSONUtils.data("cokey", soilResult.cokey));
        allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
        allModelData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
        allModelData.put(JSONUtils.data("slope_r", soilResult.slope_r));
        allModelData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100 : "NONE")));
        allModelData.put(JSONUtils.data("length", soilResult.length));
        allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
        allModelData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
        allModelData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
        allModelData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
        allModelData.put(JSONUtils.data("WaterSoilLoss", EvalResult.writeDouble(soilResult.waterErosion), "Average Annual Soil Loss by Water", "ton/ac/yr"));
        allModelData.put(JSONUtils.data("WindSoilLoss", EvalResult.writeDouble(soilResult.windErosion), "Average Annual Soil Loss by Wind", "ton/ac/yr"));
        allModelData.put(JSONUtils.data("wepp_request", aCall.serviceCall.requestJSON(), "Request JSON sent to the WEPP model"));
        allModelData.put(JSONUtils.data("wepp_result", aCall.serviceCall.getResults(), "Result JSON returned from the WEPP model"));
        subCalls.put(allModelData);
      }
      soilsDebugResults.put(JSONUtils.data("WEPP_CALLS", subCalls));

      subCalls = new JSONArray();

      for (ASYNCCall aCall : wepsCalls) {
        JSONArray allModelData = new JSONArray();
        SoilResult soilResult = aCall.soilResult;

        allModelData.put(JSONUtils.data("mukey", soilResult.mukey));
        allModelData.put(JSONUtils.data("cokey", soilResult.cokey));
        allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
        allModelData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
        allModelData.put(JSONUtils.data("slope_r", soilResult.slope_r));
        allModelData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100.0 : "NONE")));
        allModelData.put(JSONUtils.data("length", soilResult.length));
        allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
        allModelData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
        allModelData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
        allModelData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
        allModelData.put(JSONUtils.data("WaterSoilLoss", EvalResult.writeDouble(soilResult.waterErosion), "Average Annual Soil Loss by Water", "ton/ac/yr"));
        allModelData.put(JSONUtils.data("WindSoilLoss", EvalResult.writeDouble(soilResult.windErosion), "Average Annual Soil Loss by Wind", "ton/ac/yr"));
        allModelData.put(JSONUtils.data("weps_request", aCall.serviceCall.requestJSON(), "Request JSON sent to the WEPS model"));
        allModelData.put(JSONUtils.data("weps_result", aCall.serviceCall.getResults(), "Result JSON returned from the WEPS model"));
        subCalls.put(allModelData);
      }
      soilsDebugResults.put(JSONUtils.data("WEPS_CALLS", subCalls));

      results.put("DEBUGGING_DATA", soilsDebugResults);
    }
  }

  protected void getDEMSlopes(ArrayList<SoilResult> soils) throws ServiceException, SQLException, GISObjectException, JSONException, IOException {

    for (SoilResult soilResult : soils) {
      double tSlope = Double.NaN;

      slopes = getDEMSteepness(soilResult.polygonList);

      if (null != slopes) {
        if (!slopes.badSlopeData() && slopes.slopeDataMessages().isEmpty()) {
          double wAvgSlope = 0.0;
          double totalAcres = 0.0;

          //Get overall average slope for these intersected polygons from the result file.
          try (Connection connection = resources().getJDBC(LOCAL_SQLSERVER);) {
            gEngine = GISEngineFactory.createGISEngine(connection);
            for (int i = 0; i < slopes.numSlopes(); i++) {
              SlopeData tSlopeData = slopes.slope(i + 1);
              if (null != tSlopeData) {
                double tAvg = tSlopeData.mean();
                double tAcres = 0.0;
                JSONObject polyJSON = soilResult.polygonList.get(i);

                GISObject tVector = GISObjectFactory.createGISObject(polyJSON, gEngine);
                tAcres = tVector.areaInAcres();
                wAvgSlope += tAcres * tAvg;
                totalAcres += tAcres;
              }
            }
          }

          if (0.0 != totalAcres) {
            tSlope = (wAvgSlope / totalAcres);  //  Convert from rise/run to grade percentage.     
            if (tSlope < 0.01) {
              tSlope = 0.01;
            }
          } else {
            tSlope = 0.0;
            Logger.getLogger(ServiceCall.class.getName()).log(Level.WARNING, "Total acres for this cokey's," + soilResult.cokey + " mapunit intersections was zero.  Setting DEM slope to 0 .");
          }
        } else {
          throw new ServiceException("The DEM returned slopes had bad data in the returned file. " + ((!slopes.slopeDataMessages().isEmpty()) ? " Bad data message: " + slopes.slopeDataMessages() : ""));
        }
      } else {
        throw new ServiceException("No data was returned from the DEM slope servcie for this cokey, " + soilResult.cokey + " and intersected mapunit polygons");
      }

      if (Double.isNaN(tSlope)) {
        throw new ServiceException("DEM slope calculation requested, but no DEM slope could be found for this soil: " + soilResult.cokey);
      }
      soilResult.slopeDEM = tSlope;
    }
  }

  //TODO finish when Jack is ready.
  protected JSONObject getRotation(int length) throws JSONException {
    return new WEPSFallowRotation(length).rotation();
  }

  protected SlopeSteepness getDEMSteepness() throws ServiceException {
    if (!demSteepnessURI.equals("NONE")) {
      DEMSteepness slopeCall = new DEMSteepness(metainfo().toString(), dem_featureCollection, demSteepnessURI);
      slopeCall.call();

      return slopeCall.slopes();
    }

    return null;
  }

  protected SlopeSteepness getDEMSteepness(ArrayList<JSONObject> intersectedPolygons) throws ServiceException {
    if (!demSteepnessURI.equals("NONE")) {
      DEMSteepness slopeCall = new DEMSteepness(metainfo().toString(), intersectedPolygons, demSteepnessURI);
      slopeCall.call();

      return slopeCall.slopes();
    }

    return null;
  }

  protected void validateFeatureCollection(GIS_FeatureCollection featureCollection) throws ServiceException {
    int featureCount = featureCollection.getFeatureCount();

    if (0 < featureCount) {
      String id;
      try {
        for (int i = 0; i < featureCount; i++) {
          if (null != (id = featureCollection.getFeatureAttribute(i, "id"))) {
            if (id.isEmpty()) {
              throw new ServiceException("Feature " + (i + 1) + " in the input JSON FeatureCollection does not have a property value for 'id'.  Cannot proceed without this value.");
            }
          } else {
            throw new ServiceException("Feature " + (i + 1) + " in the input JSON FeatureCollection does not have a property 'id'.  Cannot proceed without it.");
          }
        }
      } catch (GISObjectException ex) {
        throw new ServiceException("Could not find the 'id' attribute within the first feature in this input JSON FeatureCollection.  Cannot proceed:  " + ex.getMessage(), ex);
      }
    } else {
      throw new ServiceException("No features were found in the input JSON FeatureCollection.");
    }
  }

  protected JSONObject convertPolyToFeatureCollection(GISObject polygon_object) throws ServiceException, IOException, JSONException {
    return new JSONObject(
        " {\n"
        + "      \"name\": \"boundary\",\n"
        + "      \"value\": [{\n"
        + "        \"type\": \"FeatureCollection\",\n"
        + "        \"features\": [\n"
        + "          {\n"
        + "            \"id\": \"1\",\n"
        + "            \"type\": \"Feature\",\n"
        + "            \"properties\": {},\n"
        + "            \"geometry\":" + polygon_object.toJSON()
        + "          }\n"
        + "          ]\n"
        + "        }]"
        + "}\n"
    );
  }

  public class ASYNCCall {

    public SoilResult soilResult;
    public ServiceCall serviceCall;
    public boolean callFinished = false;

    public ASYNCCall(SoilResult _soilResult, ServiceCall _serviceCall) {
      soilResult = _soilResult;
      serviceCall = _serviceCall;
    }
  }
}