V1_0.java [src/java/m/guidata] Revision: Date:
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package m.guidata;
import csip.ModelDataService;
import csip.api.server.ServiceException;
import csip.annotations.*;
import static csip.annotations.ResourceType.*;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.PreparedStatement;
import org.sqlite.SQLiteConfig;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import java.util.concurrent.Callable;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
import java.util.Map;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.json.XML;
import com.google.gson.Gson;
import csip.Config;
import csip.SessionLogger;
import static csip.annotations.State.RELEASED;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import java.io.FileInputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import static util.WeppConstants.CONFIG_CROPS_URL;
import static util.WeppConstants.CONFIG_MANAGEMENTS_URL;
import static util.WeppConstants.CONFIG_OPERATIONS_URL;
import static util.WeppConstants.CONFIG_RESIDUES_URL;
import static util.WeppConstants.CONFIG_SOILAOI_URL;
import static util.WeppConstants.CONFIG_STRIPS_URL;
import static util.WeppConstants.CONFIG_STATECOUNTY_URL;
import static util.WeppConstants.buildConfigKey;
/**
* This service returns data that the WEPP Web GUI needs to work. It generally
* return lists or converted managements. Some data comes from a sqlite
* database.
* <p>
* The data resources referenced include:
* <ul>
* <li>wepp-gui-data.db - Has information for counties, folders in
* management.</li>
* <li>strips.json - JSON file if new strip information. Will be replaced by a
* CRLMOD service at some point. </li>
* <li>*.man - Management files that are referenced by the strips.json
* file.</li>
* </ul>
*
* @author etheller, jf, od
*/
@Name("weppdata2017")
@Description("WEPP data table CSIP REST service- version 2/27/2018")
@Path("m/weppdata/{guiVersion}")
@State(RELEASED)
@Polling(first = 2000, next = 2000)
@Resource(file = "/data/wepp-gui-data.zip", type = ARCHIVE)
// env="open_mode:1" is for a read-only connection to sqlite
@Resource(file = "jdbc:sqlite:${csip.data.dir}/wepp-gui-data.db", id = "wepp-gui-data", type = JDBC, env = "open_mode:1")
@Resource(file = "/data/strips.json", id = "strips")
@Resource(file = "/data/warm season, grass, sod forming.man", id = "s1")
@Resource(file = "/data/warm season, bunch grass.man", id = "s2")
@Resource(file = "/data/small grain, annual.man", id = "s3")
@Resource(file = "/data/Cool season, grass, sod forming.man", id = "s4")
@Resource(file = "/data/Cool season, bunch grass.man", id = "s5")
@Resource(file = "/data/extravegs.json", id = "extravegs")
@Resource(file = "/data/extraops.json", id = "extraops")
@Resource(file = "/data/extrares.json", id = "extrares")
// Output to capture
@Resource(file = "*stdout.txt *stderr.txt", type = OUTPUT)
/**
*
*/
public class V1_0 extends ModelDataService {
private String sessionWorkDir;
private String query;
private JSONArray tableJson;
private List<String> tableList;
private JSONObject tableObj;
private String stateAbbr;
private String stripName;
private String county;
private String survey;
private String soilcomp;
private String cmz;
private String path;
private String cmd;
private String csipquery;
private String lkey;
private String verify_url_str;
private String vegName;
private String operation;
private String soilName;
private boolean use_CRLMOD;
private String manageFileName;
private boolean crlmod31_format;
private String dataversion = "";
private boolean incWoodyVegs;
private boolean incExtraOps;
private String guiVersion;
static final String WEPP_KEY_QUERY = "query";
static final String WEPP_KEY_STATE = "state";
static final String KEY_PNAME = "name";
static final String COMP_NAME = "mukey";
static final String WEPP_KEY_COUNTY = "county";
static final String WEPP_KEY_SURVEY = "survey";
static final String KEY_CMZ = "cmz";
static final String KEY_PATH = "path";
static final String WEPP_KEY_LKEY = "lkey";
static final String URL = "url";
static final String WEPP_KEY_VEGETATION = "vegName"; // we can change it later to match the client side: may be "veg".but it is not a big deal.
static final String WEPP_KEY_OPERATION = "operation";
static final String SOIL_NAME = "soilName";
static final String CRLMOD = "crlmod";
static final String WEPP_KEY_VERSION = "dataversion";
private static final String RESIDUE_TABLE_NAME = "residueswp";
private static final String VEGETATIONS_TABLE_NAME = "vegetationswp";
private static final String VEGETATIONS_CRLMOD_TABLE_NAME = "vegetationscrlmod";
private static final String OPERATIONS_TABLE_NAME = "operationswp";
private static final String CONTOURS_TABLE_NAME = "contours";
private static final String STRIPS_TABLE_NAME = "strips";
private static final String COUNTIES_TABLE_NAME = "climates2015counties";
private static final String MANAGEMENTS_TABLE_NAME = "managements2";
private static final String INC_WOODY_VEGS = "incwoodyvegs";
private static final String INC_EXTRA_OPS = "incextraops";
private static final String SOILS_DB_NAME = "soils";
private static final String GET_COUNTIES = "getcounties";
private static final String GET_MANPATHS = "getmanpaths";
private static final String GET_MANLIST = "getmanlist";
private static final String GET_MANTRANSLATE = "getmantranslate";
private static final String GET_MANFROMFILE = "getmanfromfile";
private static final String GET_SOILSPOLYGON = "getsoilspolygon";
private static final String GET_CSIPCLIMATE = "getcsipclimate";
private static final String GET_OPERATIONS = "getoperations";
private static final String GET_RESIDUES = "getresidues";
private static final String GET_SOILSURVEYS = "getsoilsurveys";
private static final String GET_SOILSCOUNTY = "getsoilscounty";
private static final String GET_SOILCOMPS = "getsoilcomps";
private static final String GET_CONTOURS = "getcontours";
private static final String GET_STRIPS = "getstrips";
private static final String GET_STRIPDATA = "getstripdata";
private static final String GET_VEGETATIONS = "getvegetations";
private static final String VERIFY_URL = "verify_url";
private static final String GET_VEGETATION_DATA = "getvegetationdata";
private static final String GET_PROC_PARAMETERS = "getprocparameters"; // get process parameters
private static final String GET_SOIL_COKEY = "getsoilcokey";
//String stripsServiceURL = Config.getString("wepp.strips","http://csip.engr.colostate.edu:8083/csip-crlmod/d/strip/3.0");
//String locServiceURL = Config.getString("wepp.location","http://csip.engr.colostate.edu:8083/csip-misc/d/stateCounty/2.0");
//String soilsAOIServiceURL = Config.getString("wepp.soilAOI","http://csip.engr.colostate.edu:8083/csip-soils/d/wwesoilparams/1.0");
//String cropsCRLMODServiceURL = Config.getString("wepp.crops","http://csip.engr.colostate.edu:8083/csip-crlmod/d/crop/3.1");
//String operationsCRLMODServiceURL = Config.getString("wepp.operations","http://csip.engr.colostate.edu:8083/csip-crlmod/d/operation/3.1");
//String residuesCRLMODServiceURL = Config.getString("wepp.residues","http://csip.engr.colostate.edu:8083/csip-crlmod/d/residue/3.1");
//String manCRLMODServiceURL = Config.getString("wepp.managements","http://csip.engr.colostate.edu:8083/csip-crlmod/d/management/3.1");
protected String cropsCRLMODServiceURL = null;
protected String operationsCRLMODServiceURL = null;
protected String residuesCRLMODServiceURL = null;
protected String manCRLMODServiceURL = null;
//protected String stripsServiceURL = null;
protected String locServiceURL = null;
protected String soilsAOIServiceURL = null;
protected String cropsCRLMODServiceURLalt = null;
protected String operationsCRLMODServiceURLalt = null;
private static final String NRCS_SSURGO_URL = "https://SDMDataAccess.sc.egov.usda.gov/Tabular/post.rest";
private static final String TRANSLATION_TABLE_NAME = "translation";
private static volatile Connection sqliteConnectiondd = null;
private static final Object sqliteConnectionWriteLockdd = new Object();
private Callable<Connection> weppData;
public V1_0(@PathParam("guiVersion") String ver) {
guiVersion = ver;
// Versions should match the WEPP versions 2, 3, 3.1, 4 in order to refer to
// the correct services.
if (ver.equals("1.0")) {
guiVersion = "3.1"; // original verison had a 1.0 but no WEPP version was 1.0
// move this a valid version
}
if (guiVersion.endsWith(".0")) {
guiVersion = guiVersion.substring(0, guiVersion.length() - 2);
}
}
/**
* Initialize sqlite database connection.
*
* @throws ServiceException any problem connecting to sqlite
* @return true if connected to sqlite.
*/
private boolean init() throws ServiceException {
LOG.info("GUIVersion=" + guiVersion + "<");
locServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_STATECOUNTY_URL));
soilsAOIServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_SOILAOI_URL));
manCRLMODServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_MANAGEMENTS_URL));
//stripsServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_STRIPS_URL));
cropsCRLMODServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_CROPS_URL));
operationsCRLMODServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_OPERATIONS_URL));
residuesCRLMODServiceURL = getConfigString(buildConfigKey(guiVersion, CONFIG_RESIDUES_URL));
// These are alternate URLs for testing CRLMOD data without going to the regular CRLMOD
// databases. There is no alternate site then it just defaults to the base services
// defined above.
cropsCRLMODServiceURLalt = Config.getString("wepp.cropscr", cropsCRLMODServiceURL);
operationsCRLMODServiceURLalt = Config.getString("wepp.operationscr", operationsCRLMODServiceURL);
crlmod31_format = true;
if (manCRLMODServiceURL.endsWith("/2.0")) {
crlmod31_format = false;
}
if (manCRLMODServiceURL.endsWith("/3.0")) {
crlmod31_format = false;
}
if (manCRLMODServiceURL.endsWith("/4.0")) {
crlmod31_format = false;
}
weppData = () -> resources().getJDBC("wepp-gui-data");
return true;
}
protected static String getConfigString(String key) throws ServiceException {
String ret_val = Config.getString(key);
if (null == ret_val) {
throw new ServiceException("Configuration parameter, '" + key + "', is missing. Cannot continue.");
}
return ret_val;
}
/**
* Get any parameters from the service request.
*
* @throws ServiceException any problems send error message in response.
*/
private void loadRequiredParameters() throws ServiceException {
stripName = null;
stateAbbr = null;
county = null;
survey = null;
soilcomp = null;
cmz = null;
path = null;
cmd = null;
verify_url_str = null;
vegName = null;
operation = null;
use_CRLMOD = false;
manageFileName = "";
incWoodyVegs = false;
incExtraOps = false;
JSONObject req;
JSONArray parms;
JSONArray parms2;
try {
// get run options
query = parameter().getString(WEPP_KEY_QUERY);
switch (query) {
case GET_VEGETATIONS:
case GET_OPERATIONS:
case GET_RESIDUES:
case GET_CONTOURS:
case GET_STRIPS:
if (parameter().has(CRLMOD)) {
use_CRLMOD = true;
}
if (parameter().has(INC_WOODY_VEGS)) {
incWoodyVegs = true;
}
if (parameter().has(INC_EXTRA_OPS)) {
incExtraOps = true;
}
break;
case VERIFY_URL:
if (parameter().has(URL)) {
verify_url_str = parameter().getString(URL);
}
break;
case GET_COUNTIES:
if (parameter().has(WEPP_KEY_STATE)) {
stateAbbr = parameter().getString(WEPP_KEY_STATE);
}
break;
case GET_MANPATHS:
if (parameter().has(CRLMOD)) {
use_CRLMOD = true;
}
if (parameter().has(KEY_CMZ)) {
cmz = parameter().getString(KEY_CMZ);
}
break;
case GET_MANLIST:
if (parameter().has(CRLMOD)) {
use_CRLMOD = true;
}
if (parameter().has(KEY_PATH)) {
path = parameter().getString(KEY_PATH);
}
break;
case GET_MANTRANSLATE:
if (parameter().has(CRLMOD)) {
use_CRLMOD = true;
}
req = request().getRequest();
csipquery = req.toString();
break;
case GET_MANFROMFILE:
if (parameter().has(CRLMOD)) {
use_CRLMOD = true;
}
if (parameter().has(KEY_PNAME)) {
manageFileName = parameter().getString(KEY_PNAME);
}
LOG.info("Management file name requested: >" + manageFileName + "<");
req = request().getRequest();
csipquery = req.toString();
break;
case GET_SOILSPOLYGON:
req = request().getRequest();
// remove table part of request
parms = req.getJSONArray("parameter");
parms2 = new JSONArray();
for (int i = 0; i < parms.length(); i++) {
JSONObject p = parms.getJSONObject(i);
if (p.getString("name").equals("query") == false) {
parms2.put(p);
}
}
req.put("parameter", parms2);
csipquery = req.toString();
break;
case GET_CSIPCLIMATE:
req = request().getRequest();
// remove table part of request
parms = req.getJSONArray("parameter");
parms2 = new JSONArray();
for (int i = 0; i < parms.length(); i++) {
JSONObject p = parms.getJSONObject(i);
if (p.getString("name").equals("query") == false) {
parms2.put(p);
}
}
req.put("parameter", parms2);
csipquery = req.toString();
break;
case GET_SOILSCOUNTY:
if (parameter().has(WEPP_KEY_LKEY)) {
lkey = parameter().getString(WEPP_KEY_LKEY);
}
break;
case GET_SOILSURVEYS:
if (parameter().has(WEPP_KEY_STATE)) {
stateAbbr = parameter().getString(WEPP_KEY_STATE);
}
if (parameter().has(WEPP_KEY_COUNTY)) {
county = parameter().getString(WEPP_KEY_COUNTY);
}
break;
case GET_SOILCOMPS:
if (parameter().has(COMP_NAME)) {
soilcomp = parameter().getString(COMP_NAME);
}
break;
case GET_SOIL_COKEY:
if (parameter().has(SOIL_NAME)) {
soilName = parameter().getString(SOIL_NAME);
}
if (parameter().has(WEPP_KEY_STATE)) {
stateAbbr = parameter().getString(WEPP_KEY_STATE);
}
if (parameter().has(WEPP_KEY_COUNTY)) {
county = parameter().getString(WEPP_KEY_COUNTY);
}
if (parameter().has(WEPP_KEY_SURVEY)) {
survey = parameter().getString(WEPP_KEY_SURVEY);
}
break;
case GET_STRIPDATA:
if (parameter().has(KEY_PNAME)) {
stripName = parameter().getString(KEY_PNAME);
}
break;
case GET_PROC_PARAMETERS:
if (parameter().has(WEPP_KEY_OPERATION)) {
operation = parameter().getString(WEPP_KEY_OPERATION);
}
if (parameter().has(WEPP_KEY_VEGETATION)) {
vegName = parameter().getString(WEPP_KEY_VEGETATION);
}
if (parameter().has(WEPP_KEY_VERSION)) {
dataversion = parameter().getString(WEPP_KEY_VERSION);
}
break;
}
} catch (Exception e) {
throw new ServiceException("WEPP error: error processing model run parameters.");
}
}
/**
* Get input parameters from WEPP request.
*
* @throws Exception getting parameters from JSON request.
*/
@Override
protected void preProcess() throws Exception {
init();
loadRequiredParameters();
}
/**
* Calls functions to build the WEPP input files and then run WEPP
*
* @throws Exception any problems completing the request.
*/
@Override
protected void doProcess() throws Exception {
tableJson = null;
tableList = null;
tableObj = null;
JSONArray results;
switch (query) {
case VERIFY_URL:
results = verifyURL(verify_url_str);
tableJson = results;
break;
case GET_VEGETATIONS:
results = loadVegetations();
tableJson = results;
break;
case GET_OPERATIONS:
results = loadOperations();
tableJson = results;
break;
case GET_RESIDUES:
results = loadResidueVegetations();
tableJson = results;
break;
case GET_CONTOURS:
results = loadContours();
tableJson = results;
break;
case GET_STRIPS:
results = loadStrips();
tableJson = results;
break;
case GET_STRIPDATA:
results = loadStripData(stripName);
tableJson = results;
break;
case GET_SOILSURVEYS:
results = loadSoilSurveys(stateAbbr, county);
tableJson = results;
break;
case GET_COUNTIES:
results = loadCounties(stateAbbr);
tableJson = results;
break;
case GET_CSIPCLIMATE:
results = loadCSIPClimate();
tableJson = results;
break;
case GET_SOILSPOLYGON:
results = loadCSIPSoilPolygon();
tableJson = results;
break;
case GET_SOILCOMPS:
results = loadSoilComp(soilcomp);
tableJson = results;
break;
case GET_SOIL_COKEY:
results = loadSoilCokeyFromName(soilName, stateAbbr, county, survey);
tableJson = results;
break;
case GET_SOILSCOUNTY:
results = loadSoilNames(lkey);
tableJson = results;
break;
case GET_MANPATHS:
results = loadPathsForCMZ(cmz);
tableJson = results;
break;
case GET_MANLIST:
List<String> mans = loadNamesForPath(path);
tableList = mans;
break;
case GET_MANTRANSLATE:
JSONObject manso = loadCSIPManagement();
tableObj = manso;
break;
case GET_MANFROMFILE:
JSONObject managefile = loadCSIPManagementFromFile(manageFileName);
tableObj = managefile;
break;
case GET_PROC_PARAMETERS:
JSONArray params = loadProcParameters(operation);
if (vegName != null) {
JSONObject vegData = loadVegetationData(vegName);
combineVegToParam(vegData, params);
}
tableJson = params;
break;
}
}
/**
* Replace vegData to plant JSONobject in params if the plan JSONObject is
* empty, otherwise append with key of "crop".
*
* @param vegData new vegetation data to insert
* @param params parameters to search through
* @throws JSONException if any
*/
private void combineVegToParam(JSONObject vegData, JSONArray params) throws JSONException {
boolean useInitialPlant = false;
int plantIndex = 0;
String usename = "";
for (int i = 0; i < params.length(); i++) {
JSONObject temp = params.optJSONObject(i);
if ((temp != null) && temp.has("plant")) {
plantIndex = i;
break;
}
if ( (temp != null) && temp.has("initialplant")) {
plantIndex = i;
useInitialPlant = true;
break;
}
}
JSONObject plant = params.optJSONObject(plantIndex);
if (useInitialPlant == false) {
usename = "plant";
} else {
usename = "initialplant";
}
JSONObject plantData = plant.optJSONObject(usename);
JSONObject name = plantData.optJSONObject("name");
if (plantData.length() == 0 || name == null || name.length() == 0) {
plant.put(usename, vegData); // replace the empty plant JSONObject with vegData
} else { //in case the plan json object is not empty.
JSONObject crop = new JSONObject();
crop.put("crop", vegData);
params.put(crop);
}
}
/**
* Get all the parameters associated with any operation. This is to support
* the information icon in the management editor.
*
* @param operation name of operation to get
* @return array of all parameters
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadProcParameters(String operation) throws JSONException, SQLException, ServiceException, Exception {
JSONArray result = new JSONArray();
String data = "";
// This is the service to get detail crop, operation and residue WEPP
// parameters, it can be different the regular CRLMOD databases for testing.
data = getBlobsCRLMOD(operationsCRLMODServiceURLalt, operation, "operation", dataversion);
try {
org.json.JSONObject temp = XML.toJSONObject(data);
String jsonStr = temp.toString(4);
System.out.println(jsonStr);
JSONObject weppOperation = new JSONObject(jsonStr);
Object res = weppOperation.getJSONObject("weppoperation").get("process");
if (res instanceof JSONArray) {
result = (JSONArray) res;
} else if (res instanceof JSONObject) {
result.put(res);
}
// append comments into result
String commentStr = weppOperation.getJSONObject("weppoperation").getString("comments");
String cleaned = cleanStr(commentStr);
JSONObject comments = new JSONObject();
comments.put("comments", cleaned);
JSONObject dataver = new JSONObject();
String verStr = weppOperation.getJSONObject("weppoperation").optString("version");
if (verStr.equals("")) {
verStr = "---";
}
dataver.put("version", verStr);
Object wepstags = weppOperation.getJSONObject("weppoperation").get("wepstags");
//JSONObject other = new JSONObject();
//other.put("wepstags", wepstags);
if (wepstags != null) {
result.put(wepstags);
}
result.put(comments);
result.put(dataver);
} catch (Exception e) {
System.err.println("ERROR occur in loadParameter(): converting XML to JSONObject.");
e.printStackTrace();
}
return result;
}
/**
* Get rid of """ in comments
*
* @param str starting string
* @return cleaned string
*/
private String cleanStr(String str) {
StringBuilder sb = new StringBuilder("");
int start = str.indexOf(""");
if (start != -1) {
start += 7; // skip to the first char
int last = str.lastIndexOf(""");
last -= 1; // skip last close single quote.
sb.append(str.subSequence(start, last));
sb.append(".");
} else {
sb.append(str);
}
return sb.toString();
}
/**
* This is the service to get detail crop, operation and residue WEPP
* parameters, it can be different the regular CRLMOD databases for testing.
* This is used in the management editor to get detail information.
*
* @param vegName name of vegetation
* @return XML parameters of vegetation from CRLMOD
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONObject loadVegetationData(String vegName) throws JSONException, SQLException, ServiceException, Exception {
JSONObject result = new JSONObject();
JSONObject wepstags = new JSONObject();
Object val;
String data = "";
data = getBlobsCRLMOD(cropsCRLMODServiceURLalt, vegName, "crop", dataversion);
try {
org.json.JSONObject temp = XML.toJSONObject(data);
String jsonStr = temp.toString(4); // indent factor : 4
JSONObject weppCrop = new JSONObject(jsonStr);
result = weppCrop.getJSONObject("weppcrop");
} catch (Exception e) {
System.err.println("ERROR occur in loadVegetationData(): converting XML to JSONObject.");
e.printStackTrace();
}
return result;
}
/**
* Get soil survey names based on state and county.
*
* @param state state to get surveys from
* @param county county to get surveys from
* @return soil surveys in the specified location
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadSoilSurveys(String state, String county) throws JSONException, SQLException, ServiceException, Exception {
JSONArray lkeyData;
JSONArray surveys = new JSONArray();
JSONObject err = new JSONObject();
err.put("Error", "Error in state, county get soils");
surveys.put(err);
String cname;
boolean hasKeys = false;
if (county.endsWith("COUNTY") || county.endsWith("PARISH")) {
int len = county.length();
cname = county.substring(0, len - 6);
cname = cname.trim();
} else if (county.endsWith("...")) {
int len = county.length();
cname = county.substring(0, len - 3);
cname = cname.trim();
} else if (county.endsWith("CITY AND BOROUGH")) {
int len = county.length();
cname = county.substring(0, len - 16);
cname = cname.trim();
} else if (county.endsWith("MUNICIPALITY")) {
int len = county.length();
cname = county.substring(0, len - 12);
cname = cname.trim();
} else if (county.endsWith("MUNICIPIO")) {
int len = county.length();
cname = county.substring(0, len - 9);
cname = cname.trim();
} else if (state.equals("VI")) {
if (county.endsWith("ISLAND")) {
int len = county.length();
cname = county.substring(0, len - 6);
cname = cname.trim();
} else {
cname = county;
}
} else if (state.equals("AS")) {
if (county.endsWith("DISTRICT")) {
int len = county.length();
cname = county.substring(0, len - 8);
cname = cname.trim();
if (cname.equals("MANU A")) { // ssurgo has a quote in the name
cname = cname.substring(0, 4);
}
} else {
cname = county;
}
} else {
cname = county;
}
// escape any ' in SQL
cname = cname.replace("'", "''");
String searchstr = cname;
String query1 = "select lkey from laoverlap where areasymbol like '" + state + "%' and areatypename = 'County or Parish' and areaname like '" + searchstr + "%'";
lkeyData = sqlPOSTQuery(query1);
String query = "";
try {
if (lkeyData != null) {
for (int i = 0; i < lkeyData.length(); i++) {
JSONObject obj = lkeyData.getJSONObject(i);
if (obj != null) {
int lkey = obj.getInt("lkey");
if (i == 0) {
query = "select lkey,areaname,areasymbol from legend where lkey='" + lkey + "' ";
hasKeys = true;
} else {
query = query + "or lkey='" + lkey + "' ";
}
}
}
if (hasKeys) {
query = query + " order by areaname";
surveys = sqlPOSTQuery(query);
} else {
err.put("Error", query1);
surveys.put(err);
}
}
} catch (Exception e) {
err.put("Error", query1);
surveys.put(err);
}
return surveys;
}
/**
* Get map unit (soil names) for a particular survey area.
* @param lkey soil survey area
* @return all soils in area
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadSoilNames(String lkey) throws JSONException, SQLException, ServiceException, Exception {
JSONArray lkeyData;
JSONArray soilNames = new JSONArray();
JSONObject err = new JSONObject();
err.put("Error", "Error in state, county get soils");
soilNames.put(err);
try {
String query = "select mukey,musym,muname from mapunit where lkey='" + lkey + "' order by musym";
soilNames = sqlPOSTQuery(query);
} catch (Exception e) {
err.put("Error", query);
soilNames.put(err);
}
return soilNames;
}
/**
* Get components within a particular map unit (soil).
*
* @param mukey map unit key
* @return soil components within map unit
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadSoilComp(String mukey) throws JSONException, SQLException, ServiceException, Exception {
JSONArray soilComps = new JSONArray();
JSONObject err = new JSONObject();
err.put("Error", "Error in state, county get soils");
soilComps.put(err);
String query = "select mukey,cokey,compname,comppct_r,tfact from component where mukey='" + mukey + "' order by compname";
try {
soilComps = sqlPOSTQuery(query);
} catch (Exception e) {
err.put("Error", query);
soilComps.put(err);
}
return soilComps;
}
/**
* This loads a soil based on the name.
*
* @param soilName soil name
* @param state state
* @param county county
* @return soil from SSURGO based only on name
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadSoilCokeyFromName(String soilName, String state, String county, String survey) throws JSONException, SQLException, ServiceException, Exception {
JSONArray soilCokeys = new JSONArray();
JSONObject err = new JSONObject();
err.put("Error", "Error in finding cokey");
soilCokeys.put(err);
String data = "{'soil by cokey error'}";
String parts[] = soilName.split("\\|"); // 0 - musym, 1-muname, 2-compname
String compname;
String cname;
if (parts.length < 3) {
throw new ServiceException("Soil Name is incomplete.");
}
if (county.endsWith("COUNTY") || county.endsWith("PARISH")) {
int len = county.length();
cname = county.substring(0, len - 6);
cname = cname.trim();
} else if (county.endsWith("...")) {
int len = county.length();
cname = county.substring(0, len - 3);
cname = cname.trim();
} else if (county.endsWith("CITY AND BOROUGH")) {
int len = county.length();
cname = county.substring(0, len - 16);
cname = cname.trim();
} else if (county.endsWith("MUNICIPALITY")) {
int len = county.length();
cname = county.substring(0, len - 12);
cname = cname.trim();
} else {
cname = county;
}
String cparts[] = parts[2].split("\\(");
compname = cparts[0].trim();
String query = null;
if (survey != null) {
if (survey.length() > 0) {
// use the survey if it exists. Saved data should now include the
// survey. Using the state and county could be problem to resolve if the
// request contained a soil map unit outside the county.
query = "select component.cokey from laoverlap inner join mapunit on laoverlap.lkey=mapunit.lkey ";
query = query + "inner join component on mapunit.mukey=component.mukey ";
query = query + "where laoverlap.areasymbol='" + survey + "' ";
query = query + "and mapunit.musym='" + parts[0] + "' and ";
query = query + "component.compname='" + compname + "'";
}
}
if (query == null) {
query = "select component.cokey from laoverlap inner join mapunit on laoverlap.lkey=mapunit.lkey ";
query = query + "inner join component on mapunit.mukey=component.mukey ";
query = query + "where laoverlap.areasymbol like '" + state + "%' and laoverlap.areatypename = 'County or Parish' ";
query = query + "and laoverlap.areaname like '" + cname + "%' and mapunit.musym='" + parts[0] + "' and ";
query = query + "component.compname='" + compname + "'";
}
LOG.info("Soil COKEY lookup: " + query + "\n");
try {
soilCokeys = sqlPOSTQuery(query);
} catch (Exception e) {
err.put("Error", query);
soilCokeys.put(err);
}
return soilCokeys;
}
/**
* Just calls the CSIP soils AOI service to get a list of soils within a
* polygon
*
* @return soils in polygon.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadCSIPSoilPolygon() throws JSONException, SQLException, ServiceException, Exception {
JSONArray csipsol;
String data;
String url = soilsAOIServiceURL;
String req = csipquery;
LOG.info("Request URL: " + url);
LOG.info("Request CSIP Polygon: " + req);
data = httpPostJson(url, req);
JSONObject st = new JSONObject(data);
JSONArray res = st.getJSONArray("result");
csipsol = res;
return csipsol;
}
/**
* Uses CSIP service to get the location (state+county) of a particular
* latitude and longitude.
*
* @return JSON results from CSIP RUSLE2 location service.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadCSIPClimate() throws JSONException, SQLException, ServiceException, Exception {
JSONArray csipcli;
String data;
String url = locServiceURL;
String req = csipquery;
LOG.info("Request URL: " + url);
LOG.info("Request CSIP Climate: " + req);
data = httpPostJson(url, req);
JSONObject st = new JSONObject(data);
JSONArray res = st.getJSONArray("result");
csipcli = res;
return csipcli;
}
/**
* Gets a CRLMOD management and translates it into the old management format.
*
* @return JSON format of original RUSLE2 management
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONObject loadCSIPManagement() throws JSONException, SQLException, ServiceException, Exception {
JSONObject csipman;
String data;
String req = csipquery;
csipman = null;
if (use_CRLMOD == true) {
String url = manCRLMODServiceURL;
LOG.info("Request URL: " + url);
LOG.info("CRLMOD REQUEST: " + req);
data = httpPostJson(url, req);
JSONObject reply = new JSONObject(data);
//LOG.info("CRLMOD RESPONSE: " + reply.toString());
csipman = new JSONObject(translate2WEPP_CRLMOD(reply, LOG));
//LOG.info("CRLMOD TRANSLATED--: " + csipman.toString());
}
return csipman;
}
/**
* Read a management file from the hard drive and return that JSON data. This
* is used in the strips/barriers managements that are not included in CRLMD.
* This should use the actual CRLMOD service to get this data when those files
* are included.
*
* @param name name of the management
* @return JSON data from management file
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONObject loadCSIPManagementFromFile(String name) throws JSONException, SQLException, ServiceException, Exception {
JSONObject csipman;
String data;
String req = csipquery;
csipman = null;
Map<String, String> map = new HashMap<String, String>();
map.put("Warm season, grass, sod forming", "s1");
map.put("Warm season, bunch grass", "s2");
map.put("Small grain, annual", "s3");
map.put("Cool season, grass, sod forming", "s4");
map.put("Cool season, bunch grass", "s5");
String id = map.get(name);
String dataFinal;
JSONArray results = new JSONArray();
File f = resources().getFile(id);
if (f.exists()) {
InputStream is = new FileInputStream(f.getPath());
String jsonTxt = IOUtils.toString(is, "UTF-8");
csipman = new JSONObject(jsonTxt);
LOG.info("loadCSIPManFromFile Data: " + csipman.toString());
is.close();
} else {
LOG.info("Error: loadCSIPManFromFile not found" + name);
}
return csipman;
}
/**
* Get all the paths to create a tree view of a particular CMZ.
*
* @param cmz CMZ to get paths for
* @return Array of paths
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadPathsForCMZ(String cmz) throws JSONException, SQLException, ServiceException, Exception {
JSONArray paths = new JSONArray();
PathElementBuilder pathElementBuilder = new PathElementBuilder();
Connection connection = getStaticConnection();
String thetable = "";
try (Statement stmt = connection.createStatement()) {
if (use_CRLMOD == true) {
thetable = "managementscrlmod";
}
try (ResultSet resultSet = stmt.executeQuery("select distinct path from " + thetable + " where path like '%CMZ " + cmz + "\\%' or path like '%Strip/Barrier Managements%' order by path")) {
String path;
while (resultSet.next()) {
path = resultSet.getString("path").trim();
if (use_CRLMOD == true) {
path = "managements\\" + path;
}
pathElementBuilder.addPath(path);
}
}
}
Gson gson = new Gson();
paths = new JSONArray(gson.toJson(pathElementBuilder.build()));
return paths;
}
/**
* Get all the management names from a specific path.
*
* @param path Path part of management name
* @return List of all managements that have path
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private List<String> loadNamesForPath(String path) throws JSONException, SQLException, ServiceException, Exception {
List<String> results = new ArrayList<>();
String thetable = "?";
Connection connection = getStaticConnection();
if (use_CRLMOD == true) {
thetable = "managementscrlmod";
path = path.substring(12);
}
try (PreparedStatement stmt = connection.prepareStatement(
"select name from " + thetable + " where path = ? order by name")) {
stmt.setString(1, path); // NOTE wildcarding here, %
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
String result = resultSet.getString("name");
results.add(result);
}
}
}
if (results.size() == 0) {
//results.add("Nothing found");
//results.add(path);
}
return results;
}
/**
* Get all the vegetations for CRLMOD. There is a limit of 100 results in CSIP
* so multiple requests are needed.
*
* @return list of all vegetations
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadVegetations() throws JSONException, SQLException, ServiceException, Exception {
JSONArray vegetations = new JSONArray();
if (use_CRLMOD == true) {
// this should be smarter to look at the first request and decide how
// many more need to be done
vegetations = loadVegetationChunk("0", vegetations);
//vegetations = loadVegetationChunk("100",vegetations);
if (incWoodyVegs) {
vegetations = addExtraVegs(vegetations);
}
}
return vegetations;
}
/**
* Load a subset of the vegetations from CRLMOD. This is because the CSIP
* results are limited to 100 entries.
*
* @param offset starting offset of where to get list
* @param vegetations existing list of vegetations
* @return new list of vegetations with results appended.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadVegetationChunk(String offset, JSONArray vegetations) throws JSONException, SQLException, ServiceException, Exception {
String url = cropsCRLMODServiceURL;
String data;
JSONArray results = new JSONArray();
JSONArray allCrops = new JSONArray();
try {
String req;
req = "{ \"metainfo\" : {},"
+ "\"parameter\": [ {\"name\": \"limit\", \"value\": \"200\"},{ \"name\": \"offset\", \"value\": \"" + offset + "\"}]}";
LOG.info("Request URL: " + url);
LOG.info("CRLMOD Vegetation REQUEST: " + req);
data = httpPostJson(url, req);
JSONObject st = new JSONObject(data);
results = st.getJSONArray("result");
JSONObject param = results.optJSONObject(0);
if (param.optString("name").equals("crlmod")) {
JSONObject paramv = param.getJSONObject("value");
allCrops = paramv.getJSONArray("crops");
} else {
LOG.info("No lmod tag");
}
} catch (Exception e) {
System.err.println("Could not get CRLMOD crop list from CSIP");
LOG.info("Could not get CRLMOD vegetatons from CSIP");
}
for (int i = 0; i < allCrops.length(); i++) {
JSONObject veg = allCrops.optJSONObject(i);
String name = veg.getString("name");
String filekey = String.valueOf(veg.getInt("id"));
String harv = veg.getString("yieldUnit");
String yld = String.valueOf(veg.getDouble("defaultYield"));
JSONObject vegetation = new JSONObject();
vegetation.put("name", name);
vegetation.put("filekey", filekey);
vegetation.put("harv", harv);
vegetation.put("yld", yld);
vegetations.put(vegetation);
}
return vegetations;
}
private JSONArray addExtraVegs(JSONArray vegetations) throws JSONException, SQLException, ServiceException, Exception {
String url = cropsCRLMODServiceURLalt;
String vegdata;
File vegfile = resources().getFile("extravegs");
JSONArray results = new JSONArray();
if (url.contains("purdue")) {
// add any local vegs that are not in the release for testing
try {
String ifile = vegfile.getAbsolutePath();
byte[] data = Files.readAllBytes(Paths.get(ifile));
vegdata = new String(data, Charset.defaultCharset());
// convert to JSONArray and append to vegs list
JSONObject st = new JSONObject(vegdata);
results = st.getJSONArray("extravegs");
for (int i = 0; i < results.length(); i++) {
JSONObject veg = results.optJSONObject(i);
String name = veg.getString("name");
String filekey = String.valueOf(veg.getInt("filekey"));
String harv = veg.getString("harv");
String yld = String.valueOf(veg.getDouble("yld"));
JSONObject vegetation = new JSONObject();
vegetation.put("name", name);
vegetation.put("filekey", filekey);
vegetation.put("harv", harv);
vegetation.put("yld", yld);
vegetations.put(vegetation);
}
} catch (Exception e) {
return vegetations;
}
}
return vegetations;
}
private JSONArray addExtraResidues(JSONArray residues) throws JSONException, SQLException, ServiceException, Exception {
String url = cropsCRLMODServiceURLalt;
String resdata;
File resfile = resources().getFile("extrares");
JSONArray results = new JSONArray();
if (url.contains("purdue")) {
// add any local vegs that are not in the release for testing
try {
String ifile = resfile.getAbsolutePath();
byte[] data = Files.readAllBytes(Paths.get(ifile));
resdata = new String(data, Charset.defaultCharset());
// convert to JSONArray and append to vegs list
JSONObject st = new JSONObject(resdata);
results = st.getJSONArray("extrares");
for (int i = 0; i < results.length(); i++) {
JSONObject veg = results.optJSONObject(i);
String name = veg.getString("name");
String filekey = String.valueOf(veg.getInt("filekey"));
String harv = veg.getString("harv");
String yld = String.valueOf(veg.getDouble("yld"));
JSONObject vegetation = new JSONObject();
vegetation.put("name", name);
vegetation.put("filekey", filekey);
vegetation.put("harv", harv);
vegetation.put("yld", yld);
residues.put(vegetation);
}
} catch (Exception e) {
return residues;
}
}
return residues;
}
/**
* Get the list of contours to display in the interface. Hardcoded for now
* until CRLMOD has a contour request.
*
* @return list of contours
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadContours() throws JSONException, SQLException, ServiceException, Exception {
JSONArray contours = new JSONArray();
// TBD - Need to get service from CSU, workaround for now 1-30-2018
JSONObject contour;
String contourNames[] = {"a. rows up-and-down hill", "c. perfect contouring no row grade",
"b. absolute row grade 0.8 percent", "b. absolute row grade 0.75 percent", "b. absolute row grade 2 percent",
"b. absolute row grade 1 percent", "b. absolute row grade 4 percent", "b. absolute row grade 3 percent",
"b. absolute row grade 0.1 percent",
"b. absolute row grade 0.25 percent", "b. absolute row grade 0.2 percent", "b. absolute row grade 0.4 percent",
"b. absolute row grade 0.3 percent", "b. absolute row grade 0.6 percent", "b. absolute row grade 0.5 percent"
};
Arrays.sort(contourNames);
for (int i = 0; i < contourNames.length; i++) {
contour = new JSONObject();
contour.put("name", contourNames[i]);
contour.put("filekey", String.valueOf(i));
contours.put(contour);
}
return contours;
}
/**
* Get list of strips/barriers. These are hardcoded for now until a CRLMOD
* strips/barriers service is available.
*
* @return list of strips/barriers
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadStrips() throws JSONException, SQLException, ServiceException, Exception {
JSONArray strips = new JSONArray();
// TBD - Need to get service from CSU
JSONObject strip;
String stripNames[] = {"0 - None",
"Cool season, bunch grass, BOTTOM",
"Cool season, bunch grass, MID-SLOPE",
"Cool season, bunch grass, TOP",
"Cool season, grass, sod forming, BOTTOM",
"Cool season, grass, sod forming, MID-SLOPE",
"Cool season, grass, sod forming, TOP",
"Small grain, annual, BOTTOM",
"Small grain, annual, MID-SLOPE",
"Small grain, annual, TOP",
"Warm season, bunch grass, BOTTOM",
"Warm season, bunch grass, MID-SLOPE",
"Warm season, bunch grass, TOP",
"Warm season, grass, sod forming, BOTTOM",
"Warm season, grass, sod forming, MID-SLOPE",
"Warm season, grass, sod forming, TOP"
};
Arrays.sort(stripNames);
for (int i = 0; i < stripNames.length; i++) {
strip = new JSONObject();
strip.put("name", stripNames[i]);
strip.put("filekey", String.valueOf(i));
strips.put(strip);
}
return strips;
}
/**
* Get a list of residues from CRLMOD. To be consistent with the other crops
* and operations functions it can piece together multiple chunks, although
* only one is needed.
*
* @return list of residues
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadResidueVegetations() throws JSONException, SQLException, ServiceException, Exception {
JSONArray residues = new JSONArray();
if (use_CRLMOD == true) {
residues = loadResidueChunk("0", residues);
}
if (incExtraOps) {
residues = addExtraResidues(residues);
}
return residues;
}
/**
* Get a list of residues from a certain offset.
*
* @param offset starting offset in residue list
* @param residues existing residue list
* @return new residue list with results appended.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadResidueChunk(String offset, JSONArray residues) throws JSONException, SQLException, ServiceException, Exception {
String url = residuesCRLMODServiceURL;
String data;
JSONArray results = new JSONArray();
JSONArray allResidues = new JSONArray();
try {
String req;
req = "{ \"metainfo\" : {},"
+ "\"parameter\": [ {\"name\": \"limit\", \"value\": \"100\"},{ \"name\": \"offset\", \"value\": \"" + offset + "\"}]}";
LOG.info("Request URL: " + url);
LOG.info("CRLMOD Residue REQUEST: " + req);
data = httpPostJson(url, req);
JSONObject st = new JSONObject(data);
results = st.getJSONArray("result");
JSONObject param = results.optJSONObject(0);
if (param.optString("name").equals("lmod")) {
JSONObject paramv = param.getJSONObject("value");
allResidues = paramv.getJSONArray("residues");
}
} catch (Exception e) {
System.err.println("Could not get CRLMOD residue list from CSIP");
LOG.info("Could not get CRLMOD residue list from CSIP");
}
for (int i = 0; i < allResidues.length(); i++) {
JSONObject res = allResidues.optJSONObject(i);
String name = res.getString("name");
String filekey = String.valueOf(res.getInt("id"));
// default for residues...
String harv = "lb/ac";
String yld = "250";
JSONObject residue = new JSONObject();
residue.put("name", name);
residue.put("filekey", filekey);
residue.put("harv", harv);
residue.put("yld", yld);
residues.put(residue);
}
return residues;
}
/**
* Get a list of all operations from CRLMOD. This is limited to 100 results
* per request so there are multiple requests to get all the data.
*
* @return list of all operations.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadOperations() throws JSONException, SQLException, ServiceException, Exception {
String url = operationsCRLMODServiceURLalt;
JSONArray ops = new JSONArray();
boolean removeIt;
if (use_CRLMOD == true) {
// this should be smarter to look at the first request and decide how
// many more need to be done
ops = getOperationChunk("0", ops);
ops = getOperationChunk("200", ops);
ops = getOperationChunk("400", ops);
//ops = getOperationChunk("300",ops);
//ops = getOperationChunk("400",ops);
}
if (incExtraOps) {
ops = removeOps(ops);
ops = addExtraOps(ops);
}
//LOG.info("OPERATIONS: " + ops.toString());
JSONArray sortedJsonArray = new JSONArray();
List<JSONObject> jsonValues = new ArrayList<JSONObject>();
for (int i = 0; i < ops.length(); i++) {
jsonValues.add(ops.getJSONObject(i));
}
Collections.sort(jsonValues, new Comparator<JSONObject>() {
//You can change "Name" with "ID" if you want to sort by ID
private static final String KEY_NAME = "name";
@Override
public int compare(JSONObject a, JSONObject b) {
String valA = new String();
String valB = new String();
try {
valA = (String) a.get(KEY_NAME);
valB = (String) b.get(KEY_NAME);
} catch (JSONException e) {
//do something
}
return valA.compareTo(valB);
//if you want to change the sort order, simply use the following:
//return -valA.compareTo(valB);
}
});
for (int i = 0; i < jsonValues.size(); i++) {
sortedJsonArray.put(jsonValues.get(i));
}
return sortedJsonArray;
}
private JSONArray removeOps(JSONArray ops) throws JSONException, ServiceException, Exception {
JSONArray allOpers = new JSONArray();
Set<String> remSet = new HashSet<String>();
remSet.add("Planter, tree, mechanical transplanter");
remSet.add("Harvest, tree, Conifer, release");
remSet.add("Harvest, biomass, fiber crop");
remSet.add("Pruning");
remSet.add("Mow, perennial vegetation ");
remSet.add("Sprayer, post emergence");
remSet.add("Manure injector, high disturb, 30 inch spacing");
remSet.add("Manure injector, low disturb, 15 inch spacing");
remSet.add("Manure injector, low disturb, 30 inch spacing");
for (int i = 0; i < ops.length(); i++) {
JSONObject op = ops.optJSONObject(i);
String name = op.getString("name");
if (!remSet.contains(name)) {
allOpers.put(op);
}
/*
if (!(name.equals("Planter, tree, mechanical transplanter") ||
name.equals("Harvest, tree, Conifer, release") ||
name.equals("Harvest, biomass, fiber crop") ||
name.equals("Pruning") ||
name.equals("Mow, perennial vegetation ") ||
name.equals("Sprayer, post emergence"))) {
allOpers.put(op);
}
*/
}
return allOpers;
}
/**
* Get upto 200 operations in a CSIP CRLMOD request.
*
* @param offset starting position in operations list
* @param ops existing list of operations
* @return new operations list with results appended.
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray getOperationChunk(String offset, JSONArray ops) throws JSONException, SQLException, ServiceException, Exception {
String url = operationsCRLMODServiceURL;
String data;
JSONArray results = new JSONArray();
JSONArray allOpers = new JSONArray();
try {
String req;
req = "{ \"metainfo\" : {},"
+ "\"parameter\": [ {\"name\": \"limit\", \"value\": \"200\"},{ \"name\": \"offset\", \"value\": \"" + offset + "\"}]}";
LOG.info("Request URL: " + url);
LOG.info("CRLMOD Operation REQUEST: " + req);
data = httpPostJson(url, req);
JSONObject st = new JSONObject(data);
results = st.getJSONArray("result");
JSONObject param = results.optJSONObject(0);
if (param.optString("name").equals("crlmod")) {
JSONObject paramv = param.getJSONObject("value");
allOpers = paramv.getJSONArray("operations");
} else {
LOG.info("No lmod tag");
}
} catch (Exception e) {
System.err.println("Could not get CRLMOD operation data from CSIP");
LOG.info("Could not get CRLMOD operation data from CSIP");
}
for (int i = 0; i < allOpers.length(); i++) {
JSONObject op = allOpers.optJSONObject(i);
String name = op.getString("name");
String id = String.valueOf(op.getInt("id"));
boolean begin_growth = op.getBoolean("begin_growth");
boolean add_residue = op.getBoolean("add_residue");
boolean kill_crop = op.getBoolean("kill_crop");
double res_amt = op.optDouble("defaultResidueAdded",0);
JSONObject nop = new JSONObject();
nop.put("name", name);
nop.put("filekey", id);
nop.put("begin_growth", begin_growth);
nop.put("add_residue", add_residue);
nop.put("kill_crop", kill_crop);
nop.put("proc", "");
nop.put("defaultResidueAdded", res_amt);
ops.put(nop);
}
return ops;
}
private JSONArray addExtraOps(JSONArray operations) throws JSONException, SQLException, ServiceException, Exception {
String url = operationsCRLMODServiceURLalt;
String opdata;
File opfile = resources().getFile("extraops");
JSONArray results = new JSONArray();
if (url.contains("purdue")) {
// add any local vegs that are not in the release for testing
try {
String ifile = opfile.getAbsolutePath();
byte[] data = Files.readAllBytes(Paths.get(ifile));
opdata = new String(data, Charset.defaultCharset());
// convert to JSONArray and append to vegs list
JSONObject st = new JSONObject(opdata);
results = st.getJSONArray("extraops");
for (int i = 0; i < results.length(); i++) {
JSONObject op = results.optJSONObject(i);
String name = op.getString("name");
String opgrp = op.getString("opGroup1");
double stir = op.getDouble("stir");
String id = String.valueOf(op.getInt("id"));
boolean begin_growth = op.getBoolean("begin_growth");
boolean add_residue = op.getBoolean("add_residue");
boolean kill_crop = op.getBoolean("kill_crop");
double defAdd = op.getDouble("defaultResidueAdded");
String notes = op.getString("opNotes");
JSONObject operation = new JSONObject();
operation.put("name", name);
//operation.put("opGroup1", opgrp);
//operation.put("stir", stir);
operation.put("filekey", id);
operation.put("begin_growth", begin_growth);
operation.put("add_residue", add_residue);
operation.put("kill_crop", kill_crop);
operation.put("defaultResidueAdded", defAdd);
operation.put("proc","");
//operation.put("opNotes", notes);
operations.put(operation);
}
} catch (Exception e) {
return operations;
}
}
return operations;
}
/**
* Get all counties for a specific state.
*
* @param state State to get counties for
* @return Counties in that state
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadCounties(String state) throws JSONException, SQLException, ServiceException, Exception {
JSONArray results = new JSONArray();
Connection connection = getStaticConnection();
try (PreparedStatement stmt = connection.prepareStatement("select station,id,lat,longitude from " + COUNTIES_TABLE_NAME + " where state=? order by station;")) {
stmt.setString(1, state);
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
JSONObject stationObject = new JSONObject();
stationObject.put("station", resultSet.getString("station"));
stationObject.put("id", resultSet.getString("id"));
stationObject.put("lat", resultSet.getString("lat"));
stationObject.put("longitude", resultSet.getString("longitude"));
results.put(stationObject);
}
}
}
return results;
}
/**
* This gets the data associated with a particular strip/barrier. Currently
* this reads the data from a file but should use a CRLMOD service in the
* future.
*
* @param name name of strip/barrier to retrieve
* @return JSON data for this strip/barrier
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray loadStripData(String name) throws JSONException, SQLException, ServiceException, Exception {
String data;
String dataFinal;
JSONArray results = new JSONArray();
File f = resources().getFile("strips");
if (f.exists()) {
InputStream is = new FileInputStream(f.getPath());
String jsonTxt = IOUtils.toString(is, "UTF-8");
JSONObject allstrips = new JSONObject(jsonTxt);
JSONArray sarr = allstrips.getJSONArray("strips");
for (int i = 0; i < sarr.length(); i++) {
JSONObject s = sarr.optJSONObject(i);
if (s.getString("name").equals(name)) {
JSONObject lm = new JSONObject();
lm.put("name", "lmod");
JSONObject s1 = new JSONObject();
JSONArray s2 = new JSONArray();
s2.put(s);
s1.put("strips", s2);
lm.put("value", s1);
results.put(0, lm);
}
}
}
return results;
}
/**
* Use for testing, makes a request to the URL and looks for a response.
*
* @param urlv URL to verify
* @return result or error message
* @throws JSONException if any
* @throws SQLException if any
* @throws ServiceException if any
* @throws Exception if any
*/
private JSONArray verifyURL(String urlv) throws JSONException, SQLException, ServiceException, Exception {
String data;
String dataFinal;
JSONArray results = new JSONArray();
JSONObject resObject = new JSONObject();
try {
StringBuilder result = new StringBuilder();
URL url = new URL(urlv);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line;
while ((line = rd.readLine()) != null) {
result.append(line);
}
rd.close();
result.toString();
resObject.put("result", result.toString());
results.put(resObject);
} catch (Exception e) {
resObject.put("result", "Error");
results.put(resObject);
}
return results;
}
/**
* Extract some WEPP outputs and include links to model input and output
* files.
*
* @throws Exception problem adding JSON to results.
*/
@Override
protected void postProcess() throws Exception {
if (tableJson != null) {
results().put("tableJson", tableJson, "A json format version of results");
} else if (tableList != null) {
//results().put("results", manlist.toArray(new String[manlist.size()]), "The list of cmzs in the given state");
//String.join("\n", tableList);
//results().put("tableJson", tableList.toArray(new String[tableList.size()]), "List array of results");
results().put("tableJson", String.join("\n", tableList), "List array of results");
} else {
results().put("tableJson", tableObj, "The translated management results");
}
}
/**
*
* This expanded checking was added by Eric Theller to prevent sqlite locking
* up with a BUSY. The key aspect is to only have one connection. If the
* connection is lost or closed another connection established.
*
* @return connection to the SQLite database
* @throws ServiceException if any
* @throws SQLException if any
* @throws Exception if any
*/
public Connection getStaticConnection() throws ServiceException, SQLException, Exception {
// BEFORE EDITING THIS CODE
// please read Java section of this Wikipedia article on Double-checked locking
// to understand why it was written exactly how it is:
// https://en.wikipedia.org/wiki/Double-checked_locking
Connection connection = sqliteConnectiondd;
if (connection == null || connection.isClosed()) {
//=============================
// Ignore netbeans warning, it was for Java SE 1.4
//
// If there is not an if statement above this text block, code
// will be about 100x slower according to Wikipedia, but NetBeans
// is bugged and if you click on the recommendation it deletes that
// line of code and makes everything 100x slower.
// You can read about how NetBeans is not correct for this warning
// unless you are using Java SE 1.4 (which was replaced by Java 1.5 in
// like 2005, and our CSIP Services currently run on Java 1.8 in 2017
// 12 years later)
// by reading this StackOverflow post:
// http://stackoverflow.com/questions/8413076/double-checked-locking-netbeans-confuses-me
//=============================
synchronized (sqliteConnectionWriteLockdd) {
// Do not remove assignment below, the static sqliteConnection
// is using Java SE 1.5+'s volatile keyword. This variable assignment
// statement, which looks like a repeat, goes back to deep memory
// again, not a cache, now it might be non-null again because of
// thread unsafety, even though we null-checked it before
connection = sqliteConnectiondd;
// If you don't make this check and then you happen to create 2
// sqliteConnections then sqlite will fail always
// with SQLITE_BUSY error. It's unlikely and only could happen
// if startup for this service began with 2 simultaneous requests,
// but if it ever did happen all requests fail with SQLITE_BUSY
// forever. So don't mess it up.
// So, we want an exterior check for speed, and an
// interior check for legitimate thread-safety so SQLITE_BUSY
// can NEVER happen.
if (connection == null || connection.isClosed()) {
SQLiteConfig sqlconfig = new SQLiteConfig();
sqlconfig.setReadOnly(true);
//sqliteConnection = connection =resources().getJDBC("wepp-gui-data");
sqliteConnectiondd = connection = weppData.call();
connection.setReadOnly(true);
}
}
}
return connection;
}
/**
* Send a HTML POST request.
*
* @param target URL to talk to
* @param content POST data to send
* @return data in response
* @throws IOException if any
*/
public static String httpPostJson(String target, String content) throws IOException {
URL url = new URL(target);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setInstanceFollowRedirects(false);
connection.setRequestMethod("POST");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("charset", "utf-8");
connection.setRequestProperty("Content-Length", "" + Integer.toString(content.getBytes().length));
connection.setUseCaches(false);
DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
wr.writeBytes(content);
wr.flush();
wr.close();
connection.disconnect();
StringBuilder output = new StringBuilder();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
String line = null;
while ((line = reader.readLine()) != null) {
output.append(line);
output.append('\n');
}
}
return output.toString();
}
/**
* Send a request to the SSURGO web services.
*
* @param query database query to execute
* @return results of query as rows
*/
private static JSONArray sqlPOSTQuery(String query) {
JSONArray output = null;
try {
String req = "{query: \"" + query + "\"}";
String resp = httpPostJson(NRCS_SSURGO_URL, req);
org.json.JSONObject temp = XML.toJSONObject(resp);
output = new JSONArray();
org.json.JSONObject dataset = temp
.optJSONObject("NewDataSet");
org.json.JSONArray table = dataset.optJSONArray("Table");
if (table != null) {
for (int i = 0; i < table.length(); i++) {
org.json.JSONObject obj = table.getJSONObject(i);
JSONObject entry = new JSONObject();
Iterator<String> keys = obj.keys();
while (keys.hasNext()) {
String s = keys.next();
if (s.contains("hasChanges")) {
continue;
}
if (s.contains(":")) {
entry.put(s.substring(s.indexOf(":") + 1), obj.get(s));
} else {
entry.put(s, obj.get(s));
}
}
output.put(entry);
}
} else {
org.json.JSONObject obj = dataset.optJSONObject("Table");
JSONObject entry = new JSONObject();
Iterator<String> keys = obj.keys();
while (keys.hasNext()) {
String s = keys.next();
if (s.contains("hasChanges")) {
continue;
}
if (s.contains(":")) {
entry.put(s.substring(s.indexOf(":") + 1), obj.get(s));
} else {
entry.put(s, obj.get(s));
}
}
output.put(entry);
}
} catch (Exception e) {
System.err.println("Error occurred while sending POST request to Server");
e.printStackTrace();
}
return output;
}
/**
* Get all operations/crops/residues parameters from CRLMOD
*
* @param url CRLMOD URL
* @param name names of objects to get
* @param type operation/crop/residue
* @return blob of data for all objects
*/
public String getBlobsCRLMOD(String url, String name, String type, String ver) {
String req;
String rnames = "";
String blob = "";
String baseName = "";
String setName = "";
String dname = "";
boolean c31_format = true;
if (url.endsWith("/2.0")) {
crlmod31_format = false;
}
if (url.endsWith("/3.0")) {
crlmod31_format = false;
}
if (url.endsWith("/4.0")) {
crlmod31_format = false;
}
if (url.indexOf("purdue") > -1) {
c31_format = true;
}
if (type.equals("operation")) {
baseName = "operations";
if (c31_format) {
setName = "weppOpFile";
} else {
setName = "weppOpParamSet";
}
// alt CRLMOD site parameter added
dname = "{ \"name\": \"dtype\", \"value\": \"operations\"},";
} else if (type.equals("crop")) {
baseName = "crops";
if (c31_format) {
setName = "weppCropFile";
} else {
setName = "weppCropParamSet";
}
// alt CRLMOD site parameter added
dname = "{ \"name\": \"dtype\", \"value\": \"vegetations\"},";
} else if (type.equals("residue")) {
baseName = "residues";
if (c31_format) {
setName = "weppResFile";
} else {
setName = "weppResParamSet";
}
// alt CRLMOD site parameter added
dname = "{ \"name\": \"dtype\", \"value\": \"residues\"},";
}
req = "{ \"metainfo\" : {},"
+ "\"parameter\": [ {\"name\": \"limit\", \"value\": \"1\"},{ \"name\": \"native_formats\", \"value\": \"true\"},";
// if the CRLMOD crops, opeerations and residues are at this site then it indicates
// alternate database for testing. It is not the full CRLMOD, just crops,
// operations and residues that can tested without messing with the real CRLMOD databases.
if (url.indexOf("purdue") > -1) {
// used for testing, alternate database
req = req + dname;
req = req + "{ \"name\": \"ver\", \"value\": \"" + ver + "\"},";
}
rnames = rnames + "\"" + name + "\"";
req = req + "{\"name\": \"name\", \"value\": [" + rnames + "]}]}";
LOG.info("URL: " + url);
LOG.info("Request: " + req);
try {
String data = httpPostJson(url, req);
JSONObject crblob = new JSONObject(data);
blob = getXMLBlob(crblob, baseName, setName, name);
} catch (Exception e) {
System.err.println("Could not get CRLMOD chunk from CSIP");
LOG.info("Error: could not get CRLMOD chunk");
}
return blob;
}
/**
* Get XML data from a detail crop/operation/residue CSIP CRLMOD response.
*
* @param obj CSIP response object
* @param baseName where to start looking in JSON different for crops/ops
* @param setName where to start looking in JSON different for crops/ops
* @param name name each subset of data
* @return string of data for that particular name
*/
public String getXMLBlob(JSONObject obj, String baseName, String setName, String name) {
String blob = "";
try {
JSONArray results = obj.getJSONArray("result");
JSONObject result0 = results.getJSONObject(0).getJSONObject("value");
JSONArray mans = result0.getJSONArray(baseName);
for (int i = 0; i < mans.length(); i++) {
JSONObject theOp = mans.getJSONObject(i);
if (theOp.getString("name").equals(name)) {
blob = theOp.getString(setName);
}
}
} catch (Exception e) {
}
if (blob.equals("")) {
LOG.info("Failed to get XML: " + name + "\n");
}
return blob;
}
/**
* Build a tree of the CRLMOD managements. This class is used when getting the
* managements by CMZ above.
*
*/
private static final class PathElement {
private String text;
private List<PathElement> nodes;
// fast lookup by name, but transient so GSON doesn't include it in JSON output
private transient Map<String, PathElement> nodeCache;
public PathElement(String text) {
this.text = text;
nodes = null;
nodeCache = new HashMap<>();
}
public void addChild(PathElement child) {
if (nodes == null) {
nodes = new ArrayList<>();
}
nodes.add(child);
nodeCache.put(child.getText(), child);
}
public PathElement getChild(String text) {
return nodeCache.get(text);
}
public String getText() {
return text;
}
public List<PathElement> getNodes() {
return nodes;
}
}
private static final class PathElementBuilder {
private PathElement root;
public PathElementBuilder() {
reset();
}
public void addPath(String path) {
String[] pathChunks = path.split("\\\\");
PathElement currentElement = root;
for (String pathChunk : pathChunks) {
final PathElement child = currentElement.getChild(pathChunk);
if (child != null) {
currentElement = child;
} else {
PathElement newPathElement = new PathElement(pathChunk);
currentElement.addChild(newPathElement);
currentElement = newPathElement;
}
}
}
public List<PathElement> build() {
PathElement result = root;
reset();
return result.getNodes();
}
private void reset() {
this.root = new PathElement("root");
}
}
/**
* Converts a CRLMOD format response into the internal structure used by WEPP
* that is basically the LMOD/RUSLE2 format.
*
* @param baseCRLMODObject object to convert
* @param LOG CSIP logging
* @return object formatted to RUSLE2/LMOD
*/
private String translate2WEPP_CRLMOD(JSONObject baseCRLMODObject, SessionLogger LOG) {
String newdata;
ArrayList<String> vegs;
ArrayList<String> ops;
ArrayList<String> residues;
ArrayList<String> dates;
ArrayList<String> ids;
ArrayList<Integer> veg_ids;
ArrayList<Integer> yields;
ArrayList<String> yieldUnits;
ArrayList<Integer> res_ids;
ArrayList<Boolean> intvs;
String path;
String name;
newdata = "";
try {
JSONArray results = baseCRLMODObject.getJSONArray("result");
JSONObject result0 = results.getJSONObject(0).getJSONObject("value");
JSONArray rots;
JSONArray mans;
if (crlmod31_format) {
rots = result0.getJSONArray("rotationFiles");
JSONObject rotsub = rots.getJSONObject(0).getJSONObject("rotation");
mans = rotsub.getJSONArray("managements");
} else {
rots = result0.getJSONArray("rotations");
mans = rots.getJSONObject(0).getJSONArray("managements");
}
//JSONArray mans = result0.getJSONArray("managements");
JSONObject baseLMODObject = mans.getJSONObject(0);
path = baseLMODObject.getString("path");
name = baseLMODObject.getString("name");
JSONArray events = baseLMODObject.getJSONArray("events");
dates = new ArrayList<>();
ids = new ArrayList<>();
ops = new ArrayList<>();
vegs = new ArrayList<>();
veg_ids = new ArrayList<>();
yields = new ArrayList<>();
yieldUnits = new ArrayList<>();
residues = new ArrayList<>();
res_ids = new ArrayList<>();
intvs = new ArrayList<>();
int yearBase = 1;
for (int i = 0; i < events.length(); i++) {
JSONObject ev = events.optJSONObject(i);
String datestr = ev.getString("date");
String parts[] = datestr.split("-");
int yr = Integer.parseInt(parts[0]);
if (yr > 1000) {
if (i == 0) {
yearBase = yr;
}
yr = (yr - yearBase) + 1;
datestr = String.format("%04d", yr) + "-" + parts[1] + "-" + parts[2];
}
dates.add(datestr);
JSONObject op = ev.getJSONObject("operation");
String idstr = op.getString("id");
ids.add(idstr);
String namestr = op.getString("name");
ops.add(namestr);
if (namestr.startsWith("Harvest")) {
intvs.add(true);
} else {
intvs.add(false);
}
JSONObject crop = ev.optJSONObject("crop");
if (crop != null) {
vegs.add(crop.getString("name"));
veg_ids.add(crop.getInt("id"));
yields.add(crop.getInt("defaultYield"));
yieldUnits.add(crop.getString("yieldUnit"));
} else {
vegs.add(null);
veg_ids.add(null);
yields.add(null);
yieldUnits.add(null);
}
JSONObject res = ev.optJSONObject("residue");
if (res != null) {
String resname = res.getString("name");
if (resname.startsWith("Residue\\")) {
resname = resname.substring(9);
}
residues.add(resname);
res_ids.add(res.getInt("id"));
} else {
res_ids.add(null);
residues.add(null);
}
}
// build the new object
JSONObject newobj = new JSONObject();
JSONObject lmod_file = new JSONObject();
JSONObject params = new JSONObject();
JSONArray param = new JSONArray();
lmod_file.put("path", path);
lmod_file.put("name", name);
// dates
JSONObject op_date = new JSONObject();
op_date.put("name", "OP_DATE");
JSONArray op_dates = new JSONArray();
for (int i = 0; i < dates.size(); i++) {
op_dates.put(dates.get(i));
}
op_date.put("data", op_dates);
param.put(op_date);
// yields
JSONObject harv_yields = new JSONObject();
harv_yields.put("name", "MAN_OP_VEG_NUM_HARV_UNITS");
JSONArray unitarr = new JSONArray();
for (int i = 0; i < yields.size(); i++) {
unitarr.put(yields.get(i));
}
harv_yields.put("data", unitarr);
param.put(harv_yields);
// operations
JSONObject my_ops = new JSONObject();
my_ops.put("name", "OP_PTR");
JSONArray oparr = new JSONArray();
for (int i = 0; i < ops.size(); i++) {
JSONObject oneop = new JSONObject();
if (ids.get(i) == null) {
oneop.put("file_key", ops.get(i));
} else {
oneop.put("file_key", ids.get(i).toString());
oneop.put("value", "operations\\" + ops.get(i));
}
oparr.put(oneop);
}
my_ops.put("data", oparr);
param.put(my_ops);
// intervals
JSONObject intv = new JSONObject();
intv.put("name","INTERVAL");
JSONArray intarr = new JSONArray();
for (int i=0;i<intvs.size();i++) {
intarr.put(intvs.get(i));
}
intv.put("data", intarr);
param.put(intv);
// crops
JSONObject my_vegs = new JSONObject();
my_vegs.put("name", "VEG_PTR");
JSONArray vegarr = new JSONArray();
for (int i = 0; i < vegs.size(); i++) {
JSONObject oneveg = new JSONObject();
if (vegs.get(i) == null) {
oneveg.put("file_key", JSONObject.NULL);
} else {
oneveg.put("file_key", veg_ids.get(i).toString());
oneveg.put("value", "vegetations\\" + vegs.get(i));
}
vegarr.put(oneveg);
}
my_vegs.put("data", vegarr);
param.put(my_vegs);
// residues
JSONObject my_ress = new JSONObject();
my_ress.put("name", "EXT_RES_PTR");
JSONArray resarr = new JSONArray();
JSONArray resaddarr = new JSONArray();
for (int i = 0; i < residues.size(); i++) {
JSONObject oneres = new JSONObject();
if (residues.get(i) == null) {
oneres.put("file_key", JSONObject.NULL);
resaddarr.put(JSONObject.NULL);
} else {
oneres.put("file_key", res_ids.get(i).toString());
oneres.put("value", "residues\\" + residues.get(i));
// there is no value in the - default to 250 lbs
resaddarr.put(250);
}
resarr.put(oneres);
}
my_ress.put("data", resarr);
param.put(my_ress);
// residue added
JSONObject resadded = new JSONObject();
resadded.put("name", "RES_ADDED");
resadded.put("data", resaddarr);
param.put(resadded);
params.put("param", param);
lmod_file.put("params", params);
newobj.put("lmod_file", lmod_file);
newdata = newobj.toString();
} catch (Exception e) {
LOG.info("translate2WEPP_CRLMOD error: could not convert management");
newdata = "";
}
return newdata;
}
}