WEPSBlobber.java [src/java/crlmod/utils] Revision: Date:
/*
* $Id: 1.0+65 WEPSBlobber.java 402d39c37049 2021-12-29 od $
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API, and application suite.
*
* 2012-2024, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package crlmod.utils;
import csip.api.server.ServiceException;
import csip.SessionLogger;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.Random;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.FileUtils;
/**
*
* @author LYaege
*/
@Deprecated
public class WEPSBlobber {
public ArrayList translations;
public String operationsPath;
public String managementsPath;
public String cropsPath;
public String transPath;
public BigInteger fileKey;
public SessionLogger LOG;
public Connection connection;
public WEPSBlobber(SessionLogger LOG, Connection connection) {
translations = new ArrayList();
fileKey = new BigInteger(63, new Random());
this.LOG = LOG;
this.connection = connection;
}
public void doImport() throws IOException, SQLException, ServiceException {
readTrans();
createCropFiles();
createOperationFiles();
}
public void createManagementFiles() throws IOException, SQLException, ServiceException {
//Create duplicate files for records in the translation file
File managementFiles = new File(managementsPath);
String[] fileNames = managementFiles.list();
for (String fileName : fileNames) {
LMODObject lmodObj = new LMODObject(fileName, "managements");
lmodObj.data = readFile(managementsPath + "/" + fileName, "man");
if (lmodObj.path.isEmpty()) {
lmodObj.path = "managements";
}
//System.out.println("Processing " + fileName);
if (NameExistsInDB(lmodObj)) {
System.out.println("Name exists!");
if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
{
updateRecord(lmodObj);
} else //Some data already exists in the db for this filename
if (DataIsIdentical(lmodObj)) {
System.out.println("Skip Record, data is identical! " + fileName);
} else //Data exists in the db, but it's different from our records.
{
updateRecord(lmodObj);
}
} else //This will be a new record, not yet in LMOD Insert
{
insertRecord(lmodObj);
System.out.println("Name Doesn't exist");
}
}
//Delete all the temporary files
File cPath = new File(managementsPath);
deleteDirectory(cPath);
}
public void createCropFiles() throws IOException, SQLException, ServiceException {
//Create duplicate files for records in the translation file
File cropFiles = new File(cropsPath);
String[] fileNames = cropFiles.list();
//Create duplicate files for records in the translation file
for (int i = 0; i < fileNames.length; ++i) {
String oldName = fileNames[i].substring(0, fileNames[i].length() - 5);
WEPSTranslationEntry e;
if (translationExists(oldName)) {
e = (WEPSTranslationEntry) translations.get(translationIndex(oldName));
String oldFilePath = cropsPath + "\\" + e.name + ".crop";
File oldFile = new File(oldFilePath);
ArrayList aliases = e.getAliases();
for (Object o : aliases) {
String name = (String) o;
String newFilePath = cropsPath + "\\" + name + ".crop";
newFilePath = newFilePath.replace("/", "@@");
File newFile = new File(newFilePath);
copyFile(oldFile, newFile);
}
}
}
fileNames = cropFiles.list();
for (String fileName : fileNames) {
LMODObject lmodObj = new LMODObject(fileName, "vegetations", "", "veg");
lmodObj.data = readFile(cropsPath + "/" + fileName, "crop");
if (lmodObj.path.isEmpty()) {
lmodObj.path = "vegetations";
}
//System.out.println("Processing " + fileName);
if (NameExistsInDB(lmodObj)) {
if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
{
updateRecord(lmodObj);
}
if (DataIsIdentical(lmodObj)) {
System.out.println("Skip Record, data is identical! " + fileName);
} else //Data exists in the db, but it's different from our records.
{
updateRecord(lmodObj);
}
} else {
//This will be a new record, not yet in LMOD Insert
//System.out.println("Insert Record: " + lmodObj.path +"\\"+ lmodObj.name);
insertRecord(lmodObj);
}
}
//Delete all the temporary files
//File cPath = new File(cropsPath);
//deleteDirectory(cPath);
}
public void createOperationFiles() throws SQLException, FileNotFoundException, IOException, ServiceException {
//Create duplicate files for records in the translation file
File opFiles = new File(operationsPath);
String[] fileNames = opFiles.list();
//Create duplicate files for records in the translation file
for (int i = 0; i < fileNames.length; ++i) {
String oldName = fileNames[i].substring(0, fileNames[i].length() - 5);
WEPSTranslationEntry e;
if (translationExists(oldName)) {
e = (WEPSTranslationEntry) translations.get(translationIndex(oldName));
String oldFilePath = operationsPath + "\\" + e.name + ".oprn";
File oldFile = new File(oldFilePath);
ArrayList aliases = e.getAliases();
for (Object o : aliases) {
String name = (String) o;
String newFilePath = operationsPath + "\\" + name + ".oprn";
newFilePath = newFilePath.replace("/", "@@");
File newFile = new File(newFilePath);
copyFile(oldFile, newFile);
}
}
}
fileNames = opFiles.list();
for (String fileName : fileNames) {
LMODObject lmodObj = new LMODObject(fileName, "operations", "", "ope");
lmodObj.data = readFile(operationsPath + "/" + fileName, "oprn");
if (lmodObj.path.isEmpty()) {
lmodObj.path = "operations";
}
//System.out.println("Processing " + fileName);
if (NameExistsInDB(lmodObj)) {
if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
;//updateRecord(lmodObj);
else //Some data already exists in the db for this filename
if (DataIsIdentical(lmodObj)) {
System.out.println("Skip Record, data is identical! " + fileName);
} else {
//Data exists in the db, but it's different from our records.
//System.out.println("Update Record: " + fileName);
//updateRecord(lmodObj);
}
} else {
//This will be a new record, not yet in LMOD Insert
insertRecord(lmodObj);
System.out.println("Insert New Record: " + fileName);
}
}
//Delete all the temporary files
//File opsPath = new File(operationsPath);
//deleteDirectory(opsPath);
}
private boolean NameExistsInDB(LMODObject lmodObj) throws SQLException, ServiceException {
boolean found;
try (
Statement statement = connection.createStatement();) {
String query = "SELECT file_name FROM file_metadata"
+ " WHERE file_name = '" + lmodObj.name
+ "' AND file_path = '" + lmodObj.path + "'";
ResultSet resultSet = statement.executeQuery(query);
found = false;
int counter = 1;
while (resultSet.next()) {
if (counter > 1) {
System.out.println("More than one matching record: " + lmodObj.path + lmodObj.name);
}
found = true;
counter++;
}
}
return found;
}
private boolean AlreadyHasData(LMODObject lmodObj) throws SQLException, ServiceException {
boolean result;
try (
Statement statement = connection.createStatement();) {
String query = "SELECT file_name FROM file_metadata"
+ " WHERE file_name = '" + lmodObj.name
+ "' AND wepsdata is not null"
+ " AND file_path = '" + lmodObj.path + "'";
ResultSet resultSet = statement.executeQuery(query);
result = resultSet.next();
}
return result;
}
private boolean DataIsIdentical(LMODObject lmodObj) throws SQLException, ServiceException {
boolean result = false;
try (
Statement statement = connection.createStatement();) {
String query = "SELECT wepsData FROM file_metadata"
+ " WHERE file_name = '" + lmodObj.name
+ "' AND file_path = '" + lmodObj.path + "'";
ResultSet resultSet = statement.executeQuery(query);
if (resultSet.next()) {
String dbData = resultSet.getString(1);
if (dbData.replaceAll("\\s+", "").equalsIgnoreCase(lmodObj.data.replaceAll("\\s+", ""))) {
result = true;
} else {
System.out.println("Data is different: " + lmodObj.path + lmodObj.name);
}
}
}
return result;
}
/**
*
* @param filePath full path to file
* @param extension file extension e.g. '.crop'
* @return The full text of the file read into a string.
* @throws FileNotFoundException
*/
private String readFile(String filePath, String extension) throws FileNotFoundException, IOException {
if (!filePath.contains("." + extension)) {
filePath += "." + extension;
}
String wepsData = FileUtils.readFileToString(new File(filePath));
return wepsData;
}
private boolean updateRecord(LMODObject lmodObj) throws SQLException, ServiceException {
try (
Statement statement = connection.createStatement();) {
String query = "UPDATE file_metadata"
+ " SET wepsdata = $$" + lmodObj.data
+ "$$ WHERE file_name = $$" + lmodObj.name
+ "$$ AND file_path = $$" + lmodObj.path + "$$";
System.out.println("Updating: " + lmodObj.path + "\\" + lmodObj.name);
//boolean updateWorked = statement.executeUpdate(query) >= 1;
//Log it to the stdout for now.
//if(!updateWorked)
// System.out.println("Update Query Failed");
//else
// System.out.println("Updated Record: " + lmodObj.path +"\\"+ lmodObj.name);
}
return true; //updateWorked;
}
private boolean insertRecord(LMODObject lmodObj) throws SQLException, ServiceException {
fileKey = fileKey.add(BigInteger.ONE);
try (Statement statement = connection.createStatement();) {
Date now = new Date();
String query = "INSERT INTO file_metadata (file_key, hash_key, hash_data, file_path, file_name, object_key, import_date, wepsdata) "
+ "VALUES (" + fileKey + ", "
+ 0 + ", "
+ 0 + ", '"
+ lmodObj.path + "', '"
+ lmodObj.name + "', \'"
+ lmodObj.objectKey + "\', \'"
+ now.toString() + "\', $$"
+ lmodObj.data + "$$)";
System.out.println("Inserting: " + lmodObj.path + "\\" + lmodObj.name + " new filekey: " + fileKey.toString());
boolean insertWorked = statement.executeUpdate(query) == 1;
//Log it to the stdout for now.
if (!insertWorked) {
System.out.println("Update query failed - Query: " + query);
} else {
System.out.println("insertRecord: " + lmodObj.path + lmodObj.name);
}
}
return false;
}
public void copyFile(File from, File to) throws IOException {
if (!from.exists()) {
System.out.println("Bad Translation File Entry - Source File doesn't exist:" + from.getName());
return;
}
if (!to.exists()) {
to.getParentFile().mkdirs();
Files.copy(from.toPath(), to.toPath());
}
}
public void unZip(String zipFile, String outputFolder) {
byte[] buffer = new byte[1024];
try {
File folder = new File(outputFolder);
if (!folder.exists()) {
folder.mkdir();
}
ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile));
ZipEntry ze = zis.getNextEntry();
while (ze != null) {
String fileName = ze.getName();
File newFile = new File(outputFolder + File.separator + fileName);
//create all non exists folders
//else you will hit FileNotFoundException for compressed folder
new File(newFile.getParent()).mkdirs();
FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
ze = zis.getNextEntry();
}
zis.closeEntry();
zis.close();
} catch (IOException ex) {
ex.printStackTrace();
}
}
public boolean deleteDirectory(File path) {
if (!path.exists()) {
System.out.println("Cannot find directory that doesn't exist.");
return false;
}
boolean ret = true;
if (path.isDirectory()) {
for (File f : path.listFiles()) {
ret = ret && deleteDirectory(f);
if (ret == false) {
System.out.println("Cannot delete file: " + f.getName());
}
}
}
return ret && path.delete();
}
public void readTrans() throws FileNotFoundException, IOException {
translations = new ArrayList();
BufferedReader br = new BufferedReader(new FileReader(transPath));
String curLine = br.readLine();
ArrayList name = new ArrayList();
ArrayList action = new ArrayList();
ArrayList newname = new ArrayList();
String[] temp;
while (curLine != null) {
if (curLine.isEmpty()) {
curLine = br.readLine();
continue;
}
if (curLine.charAt(0) != '#') {
temp = curLine.split("\\s*\\|\\s*");
if (temp[1].contains("rename")) {
if (!translationExists(temp[2])) {
WEPSTranslationEntry entry = new WEPSTranslationEntry(temp[2]);
entry.addAlias(temp[0].replace("\\", "~~"));
translations.add(entry);
} else {
int idx = translationIndex(temp[2]);
WEPSTranslationEntry entry = (WEPSTranslationEntry) translations.get(idx);
entry.addAlias(temp[0].replace("\\", "~~"));
}
}
}
curLine = br.readLine();
}
}
public boolean translationExists(String name) {
try {
for (Object o : translations) {
WEPSTranslationEntry e = (WEPSTranslationEntry) o;
if (e.name.equals(name)) {
return true;
}
}
} catch (Exception e) {
//who really cares, you failed to give me a string.
}
return false;
}
public int translationIndex(String name) {
try {
for (int i = 0; i < translations.size(); ++i) {
WEPSTranslationEntry e = (WEPSTranslationEntry) translations.get(i);
if (e.name.equals(name)) {
return i;
}
}
} catch (Exception e) {
//who really cares, you failed to give me a string.
}
return -1;
}
private class LMODObject {
public String path;
public String name;
public String data;
public String objectKey;
public LMODObject() {
}
public LMODObject(String filename, String basePath) {
if (filename.contains("~~")) {
this.path = basePath + "\\" + filename.substring(0, filename.indexOf("~~"));
//this.path = this.path.toLowerCase();
filename = filename.substring(filename.indexOf("~~") + 2, filename.length());
} else {
this.path = "";
}
if (filename.contains(".oprn") || filename.contains(".crop")) {
filename = filename.substring(0, filename.lastIndexOf("."));
}
filename = filename.replace("@@", "/");
//filename = filename.toLowerCase();
this.name = filename;
}
public LMODObject(String filename, String basePath, String data, String objectKey) {
if (filename.contains("~~")) {
this.path = basePath + "\\" + filename.substring(0, filename.indexOf("~~"));
//this.path = this.path.toLowerCase();
filename = filename.substring(filename.indexOf("~~") + 2, filename.length());
} else {
this.path = basePath;
}
if (filename.contains(".oprn") || filename.contains(".crop")) {
filename = filename.substring(0, filename.lastIndexOf("."));
}
filename = filename.replace("@@", "/");
//filename = filename.toLowerCase();
this.name = filename;
this.data = data;
this.objectKey = objectKey;
}
}
}