GenFileKeyPtrs.java [src/java/crlmod/utils] Revision: Date:
/*
* $Id: 1.0+65 GenFileKeyPtrs.java 402d39c37049 2021-12-29 od $
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API, and application suite.
*
* 2012-2024, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package crlmod.utils;
import csip.api.server.ServiceException;
import csip.SessionLogger;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.lang3.exception.ExceptionUtils;
@Deprecated
public class GenFileKeyPtrs {
/**
* @param args the command line arguments
* @param LOG the logger to log errors in the generate function.
*/
public static synchronized String generate(SessionLogger LOG, Connection con) throws SQLException {
String error = null;
String str_missingFiles = "";
HashSet<String> missingFiles = new HashSet();
try {
deleteOldFileKeyPointers(LOG, con);
genOpKeys(LOG, missingFiles, con);
genVegKeys(LOG, missingFiles, con);
genResKeys(LOG, missingFiles, con);
if (missingFiles.size() > 0) {
str_missingFiles = "The following files were referenced but not found in LMOD managements:\n";
}
for (String file : missingFiles) {
str_missingFiles += file + "\n";
}
if (!str_missingFiles.isEmpty()) {
LOG.log(Level.INFO, str_missingFiles);
}
} catch (Exception ex) {
error = ex.toString() + "\n" + ExceptionUtils.getStackTrace(ex);
}
return error;
}
private static void genOpKeys(SessionLogger LOG, HashSet missingFiles, Connection connection) throws SQLException, IOException, ServiceException {
LOG.log(Level.INFO, "START generating OP_PTR:file_keys");
String param_name = "OP_PTR";
String param_name_fk = "OP_PTR:file_key";
HashMap opKeyNameMap = new HashMap();
//get all file_keys of the right type
//establish connection
//Connection connection = null;
Statement statement = null;
ResultSet resultSet = null;
//connection = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
//Init opKeyNameMap with name/file key of all operations
statement = connection.createStatement();
String query = "SELECT file_key, file_full FROM file_metadata WHERE object_key = 'ope' AND expired_date is null";
resultSet = statement.executeQuery(query);
while (resultSet.next()) {
opKeyNameMap.put(resultSet.getString("file_full").toLowerCase(), resultSet.getString("file_key"));
}
//Loop through all the op_ptrs and make op_ptr:file_keys for them
statement = connection.createStatement();
query = "SELECT file_key, param_value, param_index "
+ "FROM file_data "
+ "WHERE param_name = 'OP_PTR' "
+ "ORDER BY file_key, param_index";
resultSet = statement.executeQuery(query);
int rowCount = 0;
//Connection con = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
Statement stmt = connection.createStatement();
stmt.execute("SELECT 1");
while (resultSet.next()) {
//for each result, add a :file_key version of the same parameter. Holding everything else the same.
String opName = resultSet.getString("param_value");
if (opName == null)
System.out.println("opName is Null!?!?");
opName = opName.replace("\\\\", "\\").toLowerCase().trim();
long file_key = resultSet.getLong("file_key");
int param_index = resultSet.getInt("param_index");
//if opName is in the key->name hash, move ahead
if (opKeyNameMap.containsKey(opName)) {
String opKey = (String) opKeyNameMap.get(opName);
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'OP_PTR:file_key', "
+ opKey + ", "
+ param_index + "); \n";
stmt.addBatch(query);
} //otherwise, log a warning.
else {
missingFiles.add(opName);
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'OP_PTR:file_key', ''"
+ ", "
+ param_index + "); \n";
stmt.addBatch(query);
}
if (rowCount % 500 == 0) {
if (rowCount % 10000 == 0) {
LOG.log(Level.INFO, "" + rowCount);
}
stmt.executeBatch();
stmt.clearBatch();
}
rowCount++;
}
stmt.executeBatch();
stmt.clearBatch();
stmt.close();
}
private static void genVegKeys(SessionLogger LOG, HashSet missingFiles, Connection connection) throws SQLException, IOException, ServiceException {
LOG.log(Level.INFO, "START generating VEG_PTR:file_keys");
HashMap vegKeyNameMap = new HashMap();
//get all file_keys of the right type
//Connection connection = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
Statement statement = connection.createStatement();
String query = "SELECT file_key, file_full FROM file_metadata WHERE object_key = 'veg' AND expired_date is null";
ResultSet resultSet = statement.executeQuery(query);
while (resultSet.next()) {
vegKeyNameMap.put(resultSet.getString("file_full").toLowerCase().trim().replace("vegetations\\", ""), resultSet.getString("file_key"));
}
statement = connection.createStatement();
query = "SELECT file_key, param_value, param_index "
+ "FROM file_data WHERE param_name = 'VEG_PTR' "
+ "ORDER BY file_key, param_index";
resultSet = statement.executeQuery(query);
int rowCount = 0;
//Connection con = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
Statement stmt = connection.createStatement();
while (resultSet.next()) {
//for each result, add a :file_key version of the same parameter. Holding everything else the same.
String vegName = resultSet.getString("param_value");
vegName = vegName.replace("\\\\", "\\").toLowerCase().trim();
long file_key = resultSet.getLong("file_key");
int param_index = resultSet.getInt("param_index");
//if vegName is in the key->name hash, move ahead
if (vegKeyNameMap.containsKey(vegName)) {
String vegKey = (String) vegKeyNameMap.get(vegName);
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'VEG_PTR:file_key', "
+ vegKey + ", "
+ param_index + "); \n";
stmt.addBatch(query);
} //otherwise, log a warning.
else {
if (!vegName.isEmpty()) {
missingFiles.add(vegName);
}
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'VEG_PTR:file_key', ''"
+ ", "
+ param_index + "); \n";
stmt.addBatch(query);
}
if (rowCount % 500 == 0) {
if (rowCount % 10000 == 0) {
LOG.log(Level.INFO, "" + rowCount);
}
stmt.executeBatch();
stmt.clearBatch();
}
rowCount++;
}
stmt.executeBatch();
stmt.clearBatch();
}
private static void genResKeys(SessionLogger LOG, HashSet missingFiles, Connection connection) throws SQLException, IOException, ServiceException {
LOG.log(Level.INFO, "START generating RES_PTR:file_keys");
HashMap resKeyNameMap = new HashMap();
//get all file_keys of the right type
//establish connection
//Connection connection = null;
Statement statement = null;
ResultSet resultSet = null;
//connection = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
//Populate the resKeyNameMap with keys and names for lookup.
statement = connection.createStatement();
String query = "SELECT file_key, file_full FROM file_metadata WHERE object_key = 'rsd' AND expired_date is null";
resultSet = statement.executeQuery(query);
while (resultSet.next()) {
resKeyNameMap.put(resultSet.getString("file_full").toLowerCase().trim(), resultSet.getString("file_key"));
}
//Get all non :file_key res PTRS from the db.
statement = connection.createStatement();
query = "SELECT file_key, param_value, param_index \n"
+ "FROM file_data \n"
+ "WHERE param_name = 'EXT_RES_PTR' \n"
+ "ORDER BY file_key, param_index";
resultSet = statement.executeQuery(query);
//Iterate through each Ptr
int rowCount = 0;
//Connection con = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
Statement stmt = connection.createStatement();
while (resultSet.next()) {
//for each result, add a :file_key version of the same parameter. Holding everything else the same.
String resName = resultSet.getString("param_value");
if (resName == null || resName.equals("null")) {
continue;
}
resName = resName.replace("\\\\", "\\").toLowerCase().trim();
long file_key = resultSet.getLong("file_key");
int param_index = resultSet.getInt("param_index");
//if resName is in the key->name hash, move ahead
if (resKeyNameMap.containsKey(resName)) {
String resKey = (String) resKeyNameMap.get(resName);
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'EXT_RES_PTR:file_key', "
+ resKey + ", "
+ param_index + "); \n";
stmt.addBatch(query);
} //otherwise, log a warning.
else {
if (resName != null && !resName.equals("null")) {
missingFiles.add(resName);
}
query = "INSERT INTO file_data(file_key, param_name, param_value, param_index) "
+ "VALUES (" + file_key
+ ", 'EXT_RES_PTR:file_key', ''"
+ ", "
+ param_index + "); \n";
stmt.addBatch(query);
}
if (rowCount % 500 == 0) {
if (rowCount % 10000 == 0) {
LOG.log(Level.INFO, "" + rowCount);
}
stmt.executeBatch();
stmt.clearBatch();
}
rowCount++;
}
stmt.executeBatch();
stmt.clearBatch();
}
private static void deleteOldFileKeyPointers(SessionLogger LOG, Connection con) throws SQLException, IOException, ServiceException {
//Connection con = lmod.utils.LMODTools.getConnection("GenFileKeyPtrs", LOG);
Statement stmt = con.createStatement();
String query = "DELETE FROM file_data "
+ "WHERE param_name = 'VEG_PTR:file_key' "
+ "OR param_name = 'OP_PTR:file_key' "
+ "OR param_name = 'EXT_RES_PTR:file_key'; \n";
stmt.addBatch(query);
stmt.executeBatch();
}
private static String readFile(File file) throws FileNotFoundException {
Scanner scanner = new Scanner(file);
String string = scanner.useDelimiter("\\Z").next();
scanner.close();
return string;
}
}