V1_0.java [src/java/m/crp/assessmenttool] Revision: default Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API, and application suite.
*
* 2012-2019, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package m.crp.assessmenttool;
import crp.utils.DBResources;
import static crp.utils.DBResources.LOCAL_SQLSERVER;
import crp.utils.DEMSteepness;
import crp.utils.Region;
import crp.utils.ServiceCall;
import crp.utils.SoilResult;
import crp.utils.WEPP;
import crp.utils.WEPS;
import crp.utils.WEPSFallowRotation;
import crp.utils.WWESoilParams;
import csip.Config;
import csip.ModelDataService;
import csip.PayloadParameter;
import csip.PayloadResults;
import csip.ServiceException;
import csip.annotations.Description;
import csip.annotations.Name;
import csip.annotations.Polling;
import csip.annotations.Resource;
import csip.utils.JSONUtils;
import data.interpretors.SlopeSteepness;
import data.interpretors.SlopeSteepness.SlopeData;
import gisobjects.GISObject;
import gisobjects.GISObjectException;
import gisobjects.GISObjectFactory;
import gisobjects.db.GISEngine;
import gisobjects.db.GISEngineFactory;
import gisobjects.vector.GIS_FeatureCollection;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ws.rs.Path;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import soils.utils.EvalResult;
/**
*
* @author <a href="mailto:shaun.case@colostate.edu">Shaun Case</a>
*/
@Name("CRP Assessment")
@Description("This service consumes a JSON request containing a request identifier and CRP Offer geometry and returns sheet/rill water and wind erosion rates for a fallow management for up to the three most dominant soil components in a CRP Offer area, plus a weighted average for each erosion rate.")
@Path("m/crpassessment/1.0")
@Polling(first = 2000, next = 1000)
@Resource(from = DBResources.class)
public class V1_0 extends ModelDataService {
public static final String SCENARIO_ID = "scenario_id";
public static final String SCENARIO_GEOMETRY = "scenario_geometry";
public static final String USE_DEM_SLOPE_SERVICE = "use_dem_slope";
public static final String SHOW_DEBUG_OUTPUT = "show_debug_output";
public static final String COMPARE_SLOPE_METHODS = "compare_slope_methods";
protected static final int SLEEP_TIMER = 500; //milliseconds to sleep while polling async model calls
protected static final int INITIAL_POLL_WAIT_TIMER = 1000; //milliseconds to sleep while polling async model calls
protected static final int GET_CALL_WAIT = 100; // milliseconds to sleep between internal get requests in loop of testing status of all calls at once.
protected JSONObject scenario_geometry;
protected String scenario_id;
protected GISObject aoa_geometry;
protected JSONObject dem_featureCollection;
protected GISEngine gEngine;
protected String regionURI = "http://csip.engr.colostate.edu:8083/csip-weps/m/region/2.0";
protected String soilsURI = "http://csip.engr.colostate.edu:8092/csip-soils/d/wwesoilparams/2.1";
protected String weppURI = "http://csip.engr.colostate.edu:8083/csip-wepp/m/wepp/3.1";
protected String wepsURI = "http://csip.engr.colostate.edu:8083/csip-weps/m/weps/5.2";
protected String demSteepnessURI = "http://csip.engr.colostate.edu:8087/csip-watershed/m/average_slope/3.0";
protected String outputFile;
protected boolean streamOutputFile = false;
protected double regionLength, regionWidth, regionOrientation;
protected ArrayList<SoilResult> topThreeComponents;
protected ArrayList<SoilResult> alternateSlopeTopThreeComponents;
protected double latitude, longitude;
protected JSONObject rotation;
protected double waterErosionRate = 0.0;
protected double windErosionRate = 0.0;
protected SlopeSteepness slopes;
protected boolean useDemSlope = true;
protected boolean debugOutput = false;
protected boolean compareSlopes = true;
protected ArrayList<ASYNCCall> weppCalls = new ArrayList<>();
protected ArrayList<ASYNCCall> wepsCalls = new ArrayList<>();
@Override
protected void preProcess() throws Exception {
PayloadParameter params = parameter();
soilsURI = Config.getString("crp.soils", soilsURI);
regionURI = Config.getString("crp.region", regionURI);
weppURI = Config.getString("crp.wepp", weppURI);
wepsURI = Config.getString("crp.weps", wepsURI);
demSteepnessURI = Config.getString("crp.dem.steepness", demSteepnessURI);
// Keep in this order.
compareSlopes = params.getBoolean(COMPARE_SLOPE_METHODS, false);
debugOutput = params.getBoolean(SHOW_DEBUG_OUTPUT, false) | compareSlopes;
if (params.has(SCENARIO_ID)) {
scenario_id = params.getString(SCENARIO_ID);
} else {
throw new ServiceException("A required input 'scenario_id' was missing. Please specify a scenario_id value.");
}
if (params.has(USE_DEM_SLOPE_SERVICE)) {
useDemSlope = params.getBoolean(USE_DEM_SLOPE_SERVICE);
}
if (params.has(SCENARIO_GEOMETRY)) {
scenario_geometry = params.getParamJSON(SCENARIO_GEOMETRY);
try (Connection connection = resources().getJDBC(LOCAL_SQLSERVER);) {
gEngine = GISEngineFactory.createGISEngine(connection);
aoa_geometry = GISObjectFactory.createGISObject(scenario_geometry, gEngine);
if (aoa_geometry.getType() == GISObject.GISObjectType.featurecollection) { //May it also be a GeometryCollection??
// This is okay, just need to step through each feature, and also need
// to submit the feature collection to watershed/m/averageslope (DEM service), making
// sure that each has a field in properties with which to index it.
validateFeatureCollection((GIS_FeatureCollection) aoa_geometry);
dem_featureCollection = scenario_geometry;
} else {
// If this is a polygon/multipolygon object, need to convert
// it into a featurecollection for the averageslope (DEM service)
if ((aoa_geometry.getType() == GISObject.GISObjectType.polygon
|| aoa_geometry.getType() == GISObject.GISObjectType.multipolygon)) {
dem_featureCollection = convertPolyToFeatureCollection(aoa_geometry);
} else {
throw new ServiceException("The input geometry must be a Polygon, Multipolygon, or FeatureCollection.");
}
}
// Shortcut to getting the centroid of a shape...Built in to the GISObject source for getLat/Lon if shape is not a point.
latitude = aoa_geometry.getLatitude();
longitude = aoa_geometry.getLongitude();
}
} else {
throw new ServiceException("A required input " + SCENARIO_GEOMETRY + " was missing. Please specify a scenario_id value.");
}
}
@Override
protected void doProcess() throws Exception {
//Get all soils information for this AoI.
WWESoilParams soilsCall = new WWESoilParams(metainfo().toString(), scenario_geometry, soilsURI);
soilsCall.call();
topThreeComponents = soilsCall.getTopThree();
// Get region data
Region regionCall = new Region(metainfo().toString(), scenario_geometry, regionURI);
regionCall.call();
regionLength = regionCall.getLength();
regionWidth = regionCall.getWidth();
regionOrientation = regionCall.getOrientation();
if (useDemSlope || compareSlopes) {
getDEMSlopes(topThreeComponents);
}
if (compareSlopes) {
alternateSlopeTopThreeComponents = new ArrayList<>();
for (SoilResult soilResult : topThreeComponents) {
SoilResult tSoilResult = new SoilResult(soilResult);
alternateSlopeTopThreeComponents.add(tSoilResult);
}
}
//TODO: Currently rotations are the default fallow operation...final version of this service requires user to provide rotation.
double totalArea = 0.0;
for (SoilResult soilResult : topThreeComponents) {
soilResult.usedDEM = useDemSlope;
WEPP weppCall = new WEPP(metainfo().toString(), latitude, longitude, soilResult.cokey,
soilResult.length, ((!useDemSlope) ? soilResult.slope_r : (soilResult.slopeDEM * 100.0)), getRotation((int) soilResult.length), weppURI);
weppCall.call();
weppCalls.add(new ASYNCCall(soilResult, weppCall));
//soilResult.waterErosion = weppCall.waterErosion();
WEPS wepsCall = new WEPS(metainfo().toString(), latitude, longitude, 200,
200, regionOrientation, soilResult.cokey, getRotation(0), wepsURI);
wepsCall.call();
wepsCalls.add(new ASYNCCall(soilResult, wepsCall));
//soilResult.windErosion = wepsCall.windErosion();
totalArea += soilResult.area;
}
if (compareSlopes) {
for (SoilResult soilResult : alternateSlopeTopThreeComponents) {
soilResult.usedDEM = !useDemSlope;
WEPP weppCall = new WEPP(metainfo().toString(), latitude, longitude, soilResult.cokey,
soilResult.length, (useDemSlope ? soilResult.slope_r : (soilResult.slopeDEM * 100.0)), getRotation((int) soilResult.length), weppURI);
weppCall.call();
weppCalls.add(new ASYNCCall(soilResult, weppCall));
}
}
if (!wepsCalls.isEmpty() && !weppCalls.isEmpty()) {
boolean doneWithCalls = false;
int count = weppCalls.size() + wepsCalls.size();
// Probably should add an overall timeout value for this loop so that it
// doesn't runn forever in really odd edge cases.
boolean initialPollWaited = false;
while (!doneWithCalls) {
int waitedMills = 0;
try {
//Wait an initial period before beginning polling all the model results.
if (!initialPollWaited) {
Thread.sleep(INITIAL_POLL_WAIT_TIMER);
initialPollWaited = true;
}
for (ASYNCCall weppCall : weppCalls) {
if (!weppCall.callFinished) {
if (weppCall.serviceCall.isFinished() || weppCall.serviceCall.asyncDone()) {
weppCall.soilResult.waterErosion = ((WEPP) weppCall.serviceCall).waterErosion();
weppCall.callFinished = true;
count--;
}
Thread.sleep(GET_CALL_WAIT);
waitedMills += GET_CALL_WAIT;
}
}
for (ASYNCCall wepsCall : wepsCalls) {
if (!wepsCall.callFinished) {
if (wepsCall.serviceCall.isFinished() || wepsCall.serviceCall.asyncDone()) {
wepsCall.soilResult.windErosion = ((WEPS) wepsCall.serviceCall).windErosion();
wepsCall.callFinished = true;
count--;
}
Thread.sleep(GET_CALL_WAIT);
waitedMills += GET_CALL_WAIT;
}
}
if (waitedMills < SLEEP_TIMER) {
Thread.sleep(SLEEP_TIMER - waitedMills);
}
} catch (InterruptedException ex) {
if (count <= 0) {
doneWithCalls = true;
}
for (ASYNCCall wepsCall : wepsCalls) {
if (!wepsCall.serviceCall.isFinished()) {
wepsCall.serviceCall.cancel();
}
}
for (ASYNCCall weppCall : weppCalls) {
if (!weppCall.serviceCall.isFinished()) {
weppCall.serviceCall.cancel();
}
}
// Allow the thread to be killed...just in case.
break;
}
if (count <= 0) {
doneWithCalls = true;
}
}
if (!doneWithCalls) {
throw new ServiceException("Could not gather all of the results of the models running. " + count + " models' results were unaccounted for.");
}
} else {
throw new ServiceException("No WEPP/WEPS model runs were created. Cannot proceed.");
}
for (SoilResult soilResult : topThreeComponents) {
if (Double.isNaN(soilResult.waterErosion) || Double.isNaN(soilResult.windErosion)) {
throw new ServiceException("Not a number value encountered while trying to average wind or water erosion values.");
}
waterErosionRate += soilResult.waterErosion * (soilResult.area / totalArea);
windErosionRate += soilResult.windErosion * (soilResult.area / totalArea);
}
}
@Override
protected void postProcess() throws Exception {
PayloadResults results = results();
JSONArray soilsResult = new JSONArray();
for (SoilResult soilResult : topThreeComponents) {
JSONArray soilData = new JSONArray();
if (debugOutput) {
soilData.put(JSONUtils.data("mukey", soilResult.mukey));
soilData.put(JSONUtils.data("cokey", soilResult.cokey));
soilData.put(JSONUtils.data("soilName", soilResult.soilName));
soilData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
soilData.put(JSONUtils.data("slope_r", soilResult.slope_r));
soilData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100.0 : "NONE")));
soilData.put(JSONUtils.data("length", soilResult.length));
soilData.put(JSONUtils.data("soilName", soilResult.soilName));
soilData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
}
soilData.put(JSONUtils.data("soilPtr", soilResult.cokey));
soilData.put(JSONUtils.data("soilName", soilResult.soilName));
soilData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
soilData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
soilData.put(JSONUtils.data("WaterSoilLoss", soilResult.waterErosion, "Average Annual Soil Loss by Water", "ton/ac/yr"));
soilData.put(JSONUtils.data("WindSoilLoss", soilResult.windErosion, "Average Annual Soil Loss by Wind", "ton/ac/yr"));
soilsResult.put(soilData);
}
results.put("top_3_soils", soilsResult);
results.put("WtAvgWaterSoilLoss", waterErosionRate, "Weighted Average Annual Soil Loss by Water", "ton/ac/yr");
results.put("WtAvgWindSoilLoss", windErosionRate, "Weighted Average Annual Soil Loss by Wind", "ton/ac/yr");
results.put("WtAvgTotalSoilLoss", waterErosionRate + windErosionRate, "Weighted Average Annual Soil Loss by Water and Wind", "ton/ac/yr");
if (debugOutput) {
JSONArray soilsDebugResults = new JSONArray();
JSONArray subCalls = new JSONArray();
for (ASYNCCall aCall : weppCalls) {
JSONArray allModelData = new JSONArray();
SoilResult soilResult = aCall.soilResult;
allModelData.put(JSONUtils.data("mukey", soilResult.mukey));
allModelData.put(JSONUtils.data("cokey", soilResult.cokey));
allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
allModelData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
allModelData.put(JSONUtils.data("slope_r", soilResult.slope_r));
allModelData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100 : "NONE")));
allModelData.put(JSONUtils.data("length", soilResult.length));
allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
allModelData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
allModelData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
allModelData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
allModelData.put(JSONUtils.data("WaterSoilLoss", EvalResult.writeDouble(soilResult.waterErosion), "Average Annual Soil Loss by Water", "ton/ac/yr"));
allModelData.put(JSONUtils.data("WindSoilLoss", EvalResult.writeDouble(soilResult.windErosion), "Average Annual Soil Loss by Wind", "ton/ac/yr"));
allModelData.put(JSONUtils.data("wepp_request", aCall.serviceCall.requestJSON(), "Request JSON sent to the WEPP model"));
allModelData.put(JSONUtils.data("wepp_result", aCall.serviceCall.getResults(), "Result JSON returned from the WEPP model"));
subCalls.put(allModelData);
}
soilsDebugResults.put(JSONUtils.data("WEPP_CALLS", subCalls));
subCalls = new JSONArray();
for (ASYNCCall aCall : wepsCalls) {
JSONArray allModelData = new JSONArray();
SoilResult soilResult = aCall.soilResult;
allModelData.put(JSONUtils.data("mukey", soilResult.mukey));
allModelData.put(JSONUtils.data("cokey", soilResult.cokey));
allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
allModelData.put(JSONUtils.data("slope_method", (!soilResult.usedDEM ? "SDM slope_r value" : "DEM Slope Service")));
allModelData.put(JSONUtils.data("slope_r", soilResult.slope_r));
allModelData.put(JSONUtils.data("dem_slope", ((!Double.isNaN(soilResult.slopeDEM)) ? soilResult.slopeDEM * 100.0 : "NONE")));
allModelData.put(JSONUtils.data("length", soilResult.length));
allModelData.put(JSONUtils.data("soilName", soilResult.soilName));
allModelData.put(JSONUtils.data("soilLongName", soilResult.soilLongName));
allModelData.put(JSONUtils.data("area_pct", soilResult.area_pct, "The percentage of the component of the mapunit intersection with the CRP Offer Area", "Percent"));
allModelData.put(JSONUtils.data("area", soilResult.area, "Soil Component Area (Acres) in the CRP Offer area", "Acres"));
allModelData.put(JSONUtils.data("WaterSoilLoss", EvalResult.writeDouble(soilResult.waterErosion), "Average Annual Soil Loss by Water", "ton/ac/yr"));
allModelData.put(JSONUtils.data("WindSoilLoss", EvalResult.writeDouble(soilResult.windErosion), "Average Annual Soil Loss by Wind", "ton/ac/yr"));
allModelData.put(JSONUtils.data("weps_request", aCall.serviceCall.requestJSON(), "Request JSON sent to the WEPS model"));
allModelData.put(JSONUtils.data("weps_result", aCall.serviceCall.getResults(), "Result JSON returned from the WEPS model"));
subCalls.put(allModelData);
}
soilsDebugResults.put(JSONUtils.data("WEPS_CALLS", subCalls));
results.put("DEBUGGING_DATA", soilsDebugResults);
}
}
protected void getDEMSlopes(ArrayList<SoilResult> soils) throws ServiceException, SQLException, GISObjectException, JSONException, IOException {
for (SoilResult soilResult : soils) {
double tSlope = Double.NaN;
slopes = getDEMSteepness(soilResult.polygonList);
if (null != slopes) {
if (!slopes.badSlopeData() && slopes.slopeDataMessages().isEmpty()) {
double wAvgSlope = 0.0;
double totalAcres = 0.0;
//Get overall average slope for these intersected polygons from the result file.
try (Connection connection = resources().getJDBC(LOCAL_SQLSERVER);) {
gEngine = GISEngineFactory.createGISEngine(connection);
for (int i = 0; i < slopes.numSlopes(); i++) {
SlopeData tSlopeData = slopes.slope(i + 1);
if (null != tSlopeData) {
double tAvg = tSlopeData.mean();
double tAcres = 0.0;
JSONObject polyJSON = soilResult.polygonList.get(i);
GISObject tVector = GISObjectFactory.createGISObject(polyJSON, gEngine);
tAcres = tVector.areaInAcres();
wAvgSlope += tAcres * tAvg;
totalAcres += tAcres;
}
}
}
if (0.0 != totalAcres) {
tSlope = (wAvgSlope / totalAcres); // Convert from rise/run to grade percentage.
if (tSlope < 0.01) {
tSlope = 0.01;
}
} else {
tSlope = 0.0;
Logger.getLogger(ServiceCall.class.getName()).log(Level.WARNING, "Total acres for this cokey's," + soilResult.cokey + " mapunit intersections was zero. Setting DEM slope to 0 .");
}
} else {
throw new ServiceException("The DEM returned slopes had bad data in the returned file. " + ((!slopes.slopeDataMessages().isEmpty()) ? " Bad data message: " + slopes.slopeDataMessages() : ""));
}
} else {
throw new ServiceException("No data was returned from the DEM slope servcie for this cokey, " + soilResult.cokey + " and intersected mapunit polygons");
}
if (Double.isNaN(tSlope)) {
throw new ServiceException("DEM slope calculation requested, but no DEM slope could be found for this soil: " + soilResult.cokey);
}
soilResult.slopeDEM = tSlope;
}
}
//TODO finish when Jack is ready.
protected JSONObject getRotation(int length) throws JSONException {
return new WEPSFallowRotation(length).rotation();
}
protected SlopeSteepness getDEMSteepness() throws ServiceException {
if (!demSteepnessURI.equals("NONE")) {
DEMSteepness slopeCall = new DEMSteepness(metainfo().toString(), dem_featureCollection, demSteepnessURI);
slopeCall.call();
return slopeCall.slopes();
}
return null;
}
protected SlopeSteepness getDEMSteepness(ArrayList<JSONObject> intersectedPolygons) throws ServiceException {
if (!demSteepnessURI.equals("NONE")) {
DEMSteepness slopeCall = new DEMSteepness(metainfo().toString(), intersectedPolygons, demSteepnessURI);
slopeCall.call();
return slopeCall.slopes();
}
return null;
}
protected void validateFeatureCollection(GIS_FeatureCollection featureCollection) throws ServiceException {
int featureCount = featureCollection.getFeatureCount();
if (0 < featureCount) {
String id;
try {
for (int i = 0; i < featureCount; i++) {
if (null != (id = featureCollection.getFeatureAttribute(i, "id"))) {
if (id.isEmpty()) {
throw new ServiceException("Feature " + (i + 1) + " in the input JSON FeatureCollection does not have a property value for 'id'. Cannot proceed without this value.");
}
} else {
throw new ServiceException("Feature " + (i + 1) + " in the input JSON FeatureCollection does not have a property 'id'. Cannot proceed without it.");
}
}
} catch (GISObjectException ex) {
throw new ServiceException("Could not find the 'id' attribute within the first feature in this input JSON FeatureCollection. Cannot proceed: " + ex.getMessage(), ex);
}
} else {
throw new ServiceException("No features were found in the input JSON FeatureCollection.");
}
}
protected JSONObject convertPolyToFeatureCollection(GISObject polygon_object) throws ServiceException, IOException, JSONException {
return new JSONObject(
" {\n"
+ " \"name\": \"boundary\",\n"
+ " \"value\": [{\n"
+ " \"type\": \"FeatureCollection\",\n"
+ " \"features\": [\n"
+ " {\n"
+ " \"id\": \"1\",\n"
+ " \"type\": \"Feature\",\n"
+ " \"properties\": {},\n"
+ " \"geometry\":" + polygon_object.toJSON()
+ " }\n"
+ " ]\n"
+ " }]"
+ "}\n"
);
}
public class ASYNCCall {
public SoilResult soilResult;
public ServiceCall serviceCall;
public boolean callFinished = false;
public ASYNCCall(SoilResult _soilResult, ServiceCall _serviceCall) {
soilResult = _soilResult;
serviceCall = _serviceCall;
}
}
}