@@ -13,17 +13,17 @@ |
import crlmod.ServiceResources; |
import static crlmod.ServiceResources.*; |
import csip.ModelDataService; |
+import csip.annotations.*; |
import csip.api.server.ServiceException; |
-import csip.annotations.*; |
import java.io.File; |
import java.io.IOException; |
import java.sql.Connection; |
-import java.sql.Statement; |
+import java.sql.PreparedStatement; |
+import java.sql.SQLException; |
import java.util.ArrayList; |
import java.util.List; |
import javax.ws.rs.Path; |
import org.apache.commons.io.FileUtils; |
-import org.apache.commons.io.FilenameUtils; |
|
/** |
* |
@@ -55,86 +55,107 @@ |
} |
|
private void importWeps() throws Exception { |
- File unzipDir = new File(workspace().getDir().getAbsolutePath() + "\\WEPSCrops"); |
- File[] unZippedFiles = unzipDir.listFiles(); |
+ File[] unZippedFiles = null; |
+ File unzipDir = new File(workspace().getDir() + File.separator + "WEPSCrops"); |
+ if (unzipDir.canRead()) { |
+ unZippedFiles = unzipDir.listFiles(); |
+ } |
//Read into weps_op object |
List<Cr> wepsCrops = getCropsFromFiles(unZippedFiles); |
|
+ //TODO: Use prepared statement here which will auto-filter/adjust the Strings being inserted. |
String query = "INSERT INTO weps_crops (weps_crop_path, weps_crop_name, weps_crop_param_set)\n" |
- + "VALUES\n"; |
- for (int i = 0; i < wepsCrops.size(); ++i) { |
- query += "\n('" + wepsCrops.get(i).crop_path + "', '" |
- + wepsCrops.get(i).crop_name + "', '" |
- + wepsCrops.get(i).crop_param_set |
- .replaceAll("'", "''") |
- .replaceAll("\r\n", "' + CHAR(13) + CHAR(10) + '") + "')"; |
- if (i != wepsCrops.size() - 1) |
- query += ","; |
- else |
- query += ";"; |
- } |
+ + "VALUES (?,?,?);"; |
|
try (Connection connection = getConnection()) { |
- try (Statement statement = connection.createStatement()) { |
+ try (PreparedStatement statement = connection.prepareStatement(query)) { |
+ for (int i = 0; i < wepsCrops.size(); i++) { |
+ statement.setString(1, wepsCrops.get(i).crop_path); |
+ statement.setString(2, wepsCrops.get(i).crop_name); |
+ statement.setString(3, wepsCrops.get(i).crop_param_set); |
+ statement.addBatch(); |
+ } |
connection.setAutoCommit(false); |
- rowsAffected = statement.executeUpdate(query); |
- connection.commit(); |
- statement.clearBatch(); |
+ try { |
+ int[] rowResults = statement.executeBatch(); |
+ for (int i = 0; i < rowResults.length; i++) { |
+ if (rowResults[i] < 0) { |
+ connection.rollback(); |
+ throw new SQLException("Not all WEPS crop insert operations succeeded."); |
+ } |
+ } |
+ connection.commit(); |
+ rowsAffected = rowResults.length; |
+ } catch (SQLException ex) { |
+ connection.rollback(); |
+ throw ex; |
+ } |
} |
} |
} |
|
private void importWepp() throws Exception { |
-// File unzipDir = new File(workspace().getDir().getAbsolutePath() + "\\WEPPCrops"); |
- File unzipDir = workspace().getFile("WEPPCrops"); |
- File[] unZippedFiles = unzipDir.listFiles(); |
+ File[] unZippedFiles = null; |
+ File unzipDir = new File(workspace().getDir() + File.separator + "WEPPCrops"); |
+ if (unzipDir.canRead()) { |
+ unZippedFiles = unzipDir.listFiles(); |
+ } |
//Read into wepp_op object |
List<Cr> weppCrops = getCropsFromFiles(unZippedFiles); |
|
- String query = "INSERT INTO wepp_crops (wepp_crop_path, wepp_crop_name, wepp_crop_param_set)\n" |
- + "VALUES\n"; |
- for (int i = 0; i < weppCrops.size(); ++i) { |
- query += "\n('" + weppCrops.get(i).crop_path + "', '" |
- + weppCrops.get(i).crop_name + "', '" |
- + weppCrops.get(i).crop_param_set |
- .replaceAll("'", "''") |
- .replaceAll("\r\n", "' + CHAR(13) + CHAR(10) + '") + "')"; |
- if (i != weppCrops.size() - 1) |
- query += ","; |
- else |
- query += ";"; |
- } |
+ String query = "INSERT INTO wepp_crops (wepp_crop_path, wepp_crop_name, wepp_crop_param_set) " |
+ + "VALUES (?,?,?)"; |
|
try (Connection connection = getConnection()) { |
- try (Statement statement = connection.createStatement()) { |
+ try (PreparedStatement statement = connection.prepareStatement(query)) { |
+ for (int i = 0; i < weppCrops.size(); i++) { |
+ statement.setString(1, weppCrops.get(i).crop_path); |
+ statement.setString(2, weppCrops.get(i).crop_name); |
+ statement.setString(3, weppCrops.get(i).crop_param_set); |
+ statement.addBatch(); |
+ } |
connection.setAutoCommit(false); |
- rowsAffected = statement.executeUpdate(query); |
- connection.commit(); |
- statement.clearBatch(); |
+ try { |
+ int[] rowResults = statement.executeBatch(); |
+ for (int i = 0; i < rowResults.length; i++) { |
+ if (rowResults[i] < 0) { |
+ connection.rollback(); |
+ throw new SQLException("Not all WEPP crop insert operations succeeded."); |
+ } |
+ } |
+ connection.commit(); |
+ rowsAffected = rowResults.length; |
+ } catch (SQLException ex) { |
+ connection.rollback(); |
+ throw ex; |
+ } |
} |
} |
} |
|
public List<Cr> getCropsFromFiles(File[] cropFiles) throws IOException { |
+ String path = ""; |
List<Cr> crops = new ArrayList<>(); |
- String basePath = workspace().getDir().getPath(); |
+ if (null != cropFiles && cropFiles.length > 0) { |
+ path = cropFiles[0].getParent().replace(workspace().getDir().getPath(), ""); |
+ } |
+ |
+ //Get the path...so complicated. |
+ if (path.indexOf(File.separator, 2) != -1) { |
+ path = path.substring(path.indexOf(File.separator, 2) + 1); |
+ } else { |
+ path = ""; |
+ } |
|
for (int i = 0; i < cropFiles.length; ++i) { |
- if (cropFiles[i].isDirectory()) |
+ if (cropFiles[i].isDirectory() && !cropFiles[i].getName().startsWith(".")) { |
crops.addAll(getCropsFromFiles(cropFiles[i].listFiles())); |
- else { |
+ } else if (!cropFiles[i].isDirectory()) { |
Cr crop = new Cr(); |
- //Get the path...so complicated. |
- String path = cropFiles[i].getParent().replace(basePath, ""); |
- if (path.indexOf("\\", 2) != -1) |
- path = path.substring(path.indexOf("\\", 2) + 1); |
- else |
- path = ""; |
- //End of path craziness |
- System.out.println("[" + path + "]"); |
crop.crop_path = path; |
- crop.crop_name = FilenameUtils.getBaseName(cropFiles[i].getName()); |
+ crop.crop_name = cropFiles[i].getName().substring(0, cropFiles[i].getName().lastIndexOf(".crop")); |
crop.crop_param_set = FileUtils.readFileToString(cropFiles[i], "UTF-8"); |
+ System.out.println("Processed: [" + path + File.separator + crop.crop_name + "]"); |
crops.add(crop); |
} |
} |
@@ -45,22 +45,23 @@ |
public static final SimpleDateFormat formatter = new SimpleDateFormat("MM/dd/yy"); |
private int rowsAffected = -1; |
|
- |
@Override |
protected void doProcess() throws Exception { |
importManagements(); |
} |
|
- |
@Override |
protected void postProcess() throws Exception { |
results().put("rowsAffected", rowsAffected); |
} |
|
+ private void importManagements() throws Exception { |
+ File[] unZippedFiles = null; |
+ File unzipDir = new File(workspace().getDir() + File.separator + "WEPSManagements"); |
+ if (unzipDir.canRead()) { |
+ unZippedFiles = unzipDir.listFiles(); |
+ } |
|
- private void importManagements() throws Exception { |
- File unzipDir = workspace().getFile("WEPSManagements"); |
- File[] unZippedFiles = unzipDir.listFiles(); |
try (Connection connection = getConnection()) { |
connection.setAutoCommit(false); |
operations = Management.GetOpPointers(connection); |
@@ -70,19 +71,20 @@ |
} |
} |
|
- |
- public String importManagementFromFiles(File[] files, Connection connection) throws Exception { |
- for (int i = 0; i < files.length; ++i) { |
- if (files[i].isDirectory()) { |
- importManagementFromFiles(files[i].listFiles(), connection); |
- } else { |
- importManagementFromFile(files[i], connection); |
+ public void importManagementFromFiles(File[] files, Connection connection) throws Exception { |
+ if (null != files) { |
+ for (int i = 0; i < files.length; ++i) { |
+ if (files[i].isDirectory() && !files[i].getName().startsWith(".")) { |
+ importManagementFromFiles(files[i].listFiles(), connection); |
+ } else { |
+ if (!files[i].getName().startsWith(".") && !files[i].getName().startsWith("_")) { |
+ importManagementFromFile(files[i], connection); |
+ } |
+ } |
} |
} |
- return null; |
} |
|
- |
public void importManagementFromFile(File file, Connection connection) throws Exception { |
ArrayList<Event> events = new ArrayList<Event>(); |
|
@@ -124,17 +126,16 @@ |
//System.out.println(event); |
} |
|
- String pathPrefix = this.workspace().getDir().getAbsolutePath() + "\\WEPSManagements\\"; |
+ String pathPrefix = workspace().getDir().getPath() + File.separator + "WEPSManagements" + File.separator; |
Management management = new Management(file.getAbsolutePath().replace(pathPrefix, "")); |
management.duration = int_duration; |
management.events = events; |
management.ResolveAllPointers(operations, crops, residues); |
management.PerformInsert(connection); |
- System.out.println("Inserted: " + management.path + "\\" + management.name); |
+ System.out.println("Inserted: " + management.path + File.separator + management.name); |
rowsAffected++; |
} |
|
- |
protected Connection getConnection() throws ServiceException { |
return resources().getJDBC(CR_LMOD_2018_ID); |
} |
@@ -13,19 +13,17 @@ |
import crlmod.ServiceResources; |
import static crlmod.ServiceResources.*; |
import csip.ModelDataService; |
+import csip.annotations.*; |
import csip.api.server.ServiceException; |
-import csip.annotations.*; |
import java.io.File; |
import java.io.IOException; |
import java.sql.Connection; |
+import java.sql.PreparedStatement; |
import java.sql.SQLException; |
-import java.sql.Statement; |
import java.util.ArrayList; |
-import java.util.logging.Level; |
+import java.util.List; |
import javax.ws.rs.Path; |
import org.apache.commons.io.FileUtils; |
-import org.apache.commons.io.FilenameUtils; |
-import org.codehaus.jettison.json.JSONException; |
|
@Name("operations Import") |
@Description("Operation import service for the CRLMOD_2018 database") |
@@ -38,125 +36,129 @@ |
|
@Override |
protected void doProcess() throws Exception { |
- try { |
- String sourceModel = parameter().getString("source_model"); |
- String error = null; |
- if (sourceModel.equalsIgnoreCase("weps")) { |
- error = importWeps(); |
- } else if (sourceModel.equalsIgnoreCase("wepp")) { |
- error = importWepp(); |
- } else { |
- throw new ServiceException("required input missing 'source_model' must be included in your json."); |
- } |
- |
- if (error != null) { |
- LOG.log(Level.SEVERE, error); |
- throw new ServiceException(error); |
- } |
- } catch (SQLException | JSONException ex) { |
- LOG.log(Level.SEVERE, ex.getMessage(), ex); |
- throw new ServiceException(ex); |
+ String sourceModel = parameter().getString("source_model"); |
+ if (sourceModel.equalsIgnoreCase("weps")) { |
+ importWeps(); |
+ } else if (sourceModel.equalsIgnoreCase("wepp")) { |
+ importWepp(); |
+ } else { |
+ throw new ServiceException("required input missing 'source_model' must be included in your json."); |
} |
- } |
+ } |
|
@Override |
protected void postProcess() throws Exception { |
results().put("rowsAffected", rowsAffected); |
} |
|
- private String importWeps() throws JSONException, SQLException, ServiceException, IOException { |
- File unzipDir = workspace().getFile("WEPSOperations"); |
+ private void importWeps() throws Exception { |
+ File[] unZippedFiles = null; |
+ File unzipDir = new File(workspace().getDir() + File.separator + "WEPSOperations"); |
+ if (unzipDir.canRead()) { |
+ unZippedFiles = unzipDir.listFiles(); |
+ } |
+ //Read into weps_op object |
+ List<Op> wepsOps = getOpsFromFiles(unZippedFiles); |
|
- File[] unZippedFiles = unzipDir.listFiles(); |
- //Read into weps_op object |
- ArrayList<Op> wepsOps = getOpsFromFiles(unZippedFiles); |
+// String query = "INSERT INTO weps_ops (weps_op_path, weps_op_name, weps_op_param_set) " |
+// + "VALUES (?,?,?)"; |
|
- String query = "INSERT INTO weps_ops(weps_op_path, weps_op_name, weps_op_param_set)\n" |
- + "VALUES\n"; |
- for (int i = 0; i < wepsOps.size(); ++i) { |
- query += "\n('" + wepsOps.get(i).op_path + "','" |
- + wepsOps.get(i).op_name + "', '" |
- + wepsOps.get(i).op_param_set |
- .replaceAll("'", "''") |
- .replaceAll("\r\n", "' + CHAR(13) + CHAR(10) + '") + "')"; |
- if (i != wepsOps.size() - 1) { |
- query += ","; |
- } else { |
- query += ";"; |
+ String query = "UPDATE weps_ops SET weps_op_param_set=? WHERE weps_op_path=? AND weps_op_name=?;"; |
+ |
+ try (Connection connection = getConnection()) { |
+ try (PreparedStatement statement = connection.prepareStatement(query)) { |
+ for (int i = 0; i < wepsOps.size(); i++) { |
+ statement.setString(1, wepsOps.get(i).op_param_set); |
+ statement.setString(2, wepsOps.get(i).op_path); |
+ statement.setString(3, wepsOps.get(i).op_name); |
+ |
+ statement.addBatch(); |
+ } |
+ connection.setAutoCommit(false); |
+ try { |
+ int[] rowResults = statement.executeBatch(); |
+ for (int i = 0; i < rowResults.length; i++) { |
+ if (rowResults[i] < 0) { |
+ connection.rollback(); |
+ throw new SQLException("Not all WEPS operation insert operations succeeded."); |
+ } |
+ } |
+ connection.commit(); |
+ rowsAffected = rowResults.length; |
+ } catch (SQLException ex) { |
+ connection.rollback(); |
+ throw ex; |
+ } |
} |
+ } |
+ } |
|
+ private void importWepp() throws Exception { |
+ File[] unZippedFiles = null; |
+ File unzipDir = new File(workspace().getDir() + File.separator + "WEPPOperations"); |
+ if (unzipDir.canRead()) { |
+ unZippedFiles = unzipDir.listFiles(); |
+ } |
+ //Read into wepp_op object |
+ List<Op> weppOps = getOpsFromFiles(unZippedFiles); |
+ |
+// String query = "INSERT INTO wepp_ops (wepp_op_path, wepp_op_name, wepp_op_param_set) " |
+// + "VALUES (?,?,?)"; |
+ |
+ String query = "UPDATE wepp_ops SET wepp_op_param_set=? WHERE wepp_op_path=? AND wepp_op_name=?; " |
+ + "VALUES (?,?,?)"; |
+ |
+ try (Connection connection = getConnection()) { |
+ try (PreparedStatement statement = connection.prepareStatement(query)) { |
+ for (int i = 0; i < weppOps.size(); i++) { |
+ statement.setString(1, weppOps.get(i).op_param_set); |
+ statement.setString(2, weppOps.get(i).op_path); |
+ statement.setString(3, weppOps.get(i).op_name); |
+ |
+ statement.addBatch(); |
+ } |
+ connection.setAutoCommit(false); |
+ try { |
+ int[] rowResults = statement.executeBatch(); |
+ for (int i = 0; i < rowResults.length; i++) { |
+ if (rowResults[i] < 0) { |
+ connection.rollback(); |
+ throw new SQLException("Not all WEPP Operation insert operations succeeded."); |
+ } |
+ } |
+ connection.commit(); |
+ rowsAffected = rowResults.length; |
+ } catch (SQLException ex) { |
+ connection.rollback(); |
+ throw ex; |
+ } |
+ } |
+ } |
+ } |
+ |
+ public List<Op> getOpsFromFiles(File[] opFiles) throws IOException { |
+ String path = ""; |
+ List<Op> ops = new ArrayList<>(); |
+ if (null != opFiles && opFiles.length > 0) { |
+ path = opFiles[0].getParent().replace(workspace().getDir().getPath(), ""); |
} |
|
- try (Connection connection = getConnection()) { |
- try (Statement statement = connection.createStatement()) { |
- connection.setAutoCommit(false); |
- rowsAffected = statement.executeUpdate(query); |
- connection.commit(); |
- statement.clearBatch(); |
- statement.close(); |
- connection.close(); |
- } |
- } |
- return null; |
- } |
- |
- private String importWepp() throws JSONException, SQLException, ServiceException, IOException { |
-// File unzipDir = new File(workspace().getDir().getAbsolutePath() + "\\WEPPOperations"); |
- File unzipDir = workspace().getFile("WEPPOperations"); |
- |
- File[] unZippedFiles = unzipDir.listFiles(); |
- //Read into wepp_op object |
- ArrayList<Op> weppOps = getOpsFromFiles(unZippedFiles); |
- |
- String query = "INSERT INTO wepp_ops(wepp_op_path, wepp_op_name, wepp_op_param_set)\n" |
- + "VALUES\n"; |
- for (int i = 0; i < weppOps.size(); ++i) { |
- query += "\n('" + weppOps.get(i).op_path + "','" |
- + weppOps.get(i).op_name + "', '" |
- + weppOps.get(i).op_param_set |
- .replaceAll("'", "''") |
- .replaceAll("\r\n", "' + CHAR(13) + CHAR(10) + '") + "')"; |
- if (i != weppOps.size() - 1) { |
- query += ","; |
- } else { |
- query += ";"; |
- } |
- |
+ //Get the path...so complicated. |
+ if (path.indexOf(File.separator, 2) != -1) { |
+ path = path.substring(path.indexOf(File.separator, 2) + 1); |
+ } else { |
+ path = ""; |
} |
|
- try (Connection connection = getConnection())//resources().getJDBC(CR_LMOD_2018_ID)) |
- { |
- try (Statement statement = connection.createStatement()) { |
- connection.setAutoCommit(false); |
- rowsAffected = statement.executeUpdate(query); |
- connection.commit(); |
- statement.clearBatch(); |
- statement.close(); |
- connection.close(); |
- } |
- } |
- return null; |
- } |
- |
- public ArrayList<Op> getOpsFromFiles(File[] opFiles) throws IOException { |
- ArrayList<Op> ops = new ArrayList<>(); |
- String basePath = workspace().getDir().getPath(); |
- |
for (int i = 0; i < opFiles.length; ++i) { |
- if (opFiles[i].isDirectory()) { |
+ if (opFiles[i].isDirectory() && !opFiles[i].getName().startsWith(".")) { |
ops.addAll(getOpsFromFiles(opFiles[i].listFiles())); |
- } else { |
+ } else if (!opFiles[i].isDirectory() && !opFiles[i].getName().startsWith("_")) { |
Op op = new Op(); |
- String path = opFiles[i].getParent().replace(basePath, ""); |
- if (path.indexOf("\\", 2) != -1) { |
- path = path.substring(path.indexOf("\\", 2) + 1); |
- } else { |
- path = ""; |
- } |
- System.out.println("[" + path + "]"); |
op.op_path = path; |
- op.op_name = FilenameUtils.getBaseName(opFiles[i].getName()); |
+ op.op_name = opFiles[i].getName().substring(0, opFiles[i].getName().lastIndexOf(".oprn")); |
op.op_param_set = FileUtils.readFileToString(opFiles[i], "UTF-8"); |
+ System.out.println("Processed: [" + path + File.separator + op.op_name + "]"); |
ops.add(op); |
} |
} |