WEPSBlobber.java [src/java/crlmod/utils] Revision:   Date:
/*
 * $Id: 1.0+65 WEPSBlobber.java 402d39c37049 2021-12-29 od $
 *
 * This file is part of the Cloud Services Integration Platform (CSIP),
 * a Model-as-a-Service framework, API, and application suite.
 *
 * 2012-2024, OMSLab, Colorado State University.
 *
 * OMSLab licenses this file to you under the MIT license.
 * See the LICENSE file in the project root for more information.
 */
package crlmod.utils;

import csip.api.server.ServiceException;
import csip.SessionLogger;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.Random;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.FileUtils;

/**
 *
 * @author LYaege
 */
@Deprecated
public class WEPSBlobber {

  public ArrayList translations;
  public String operationsPath;
  public String managementsPath;
  public String cropsPath;
  public String transPath;
  public BigInteger fileKey;
  public SessionLogger LOG;
  public Connection connection;


  public WEPSBlobber(SessionLogger LOG, Connection connection) {
    translations = new ArrayList();
    fileKey = new BigInteger(63, new Random());
    this.LOG = LOG;
    this.connection = connection;
  }


  public void doImport() throws IOException, SQLException, ServiceException {
    readTrans();
    createCropFiles();
    createOperationFiles();
  }


  public void createManagementFiles() throws IOException, SQLException, ServiceException {
    //Create duplicate files for records in the translation file

    File managementFiles = new File(managementsPath);
    String[] fileNames = managementFiles.list();

    for (String fileName : fileNames) {
      LMODObject lmodObj = new LMODObject(fileName, "managements");
      lmodObj.data = readFile(managementsPath + "/" + fileName, "man");
      if (lmodObj.path.isEmpty()) {
        lmodObj.path = "managements";
      }

      //System.out.println("Processing " + fileName);
      if (NameExistsInDB(lmodObj)) {
        System.out.println("Name exists!");
        if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
        {
          updateRecord(lmodObj);
        } else //Some data already exists in the db for this filename
        if (DataIsIdentical(lmodObj)) {
          System.out.println("Skip Record, data is identical! " + fileName);
        } else //Data exists in the db, but it's different from our records.
        {
          updateRecord(lmodObj);
        }
      } else //This will be a new record, not yet in LMOD Insert
      {
        insertRecord(lmodObj);
        System.out.println("Name Doesn't exist");
      }
    }

    //Delete all the temporary files
    File cPath = new File(managementsPath);
    deleteDirectory(cPath);
  }


  public void createCropFiles() throws IOException, SQLException, ServiceException {
    //Create duplicate files for records in the translation file
    File cropFiles = new File(cropsPath);
    String[] fileNames = cropFiles.list();

    //Create duplicate files for records in the translation file
    for (int i = 0; i < fileNames.length; ++i) {
      String oldName = fileNames[i].substring(0, fileNames[i].length() - 5);
      WEPSTranslationEntry e;

      if (translationExists(oldName)) {
        e = (WEPSTranslationEntry) translations.get(translationIndex(oldName));

        String oldFilePath = cropsPath + "\\" + e.name + ".crop";
        File oldFile = new File(oldFilePath);
        ArrayList aliases = e.getAliases();

        for (Object o : aliases) {
          String name = (String) o;
          String newFilePath = cropsPath + "\\" + name + ".crop";
          newFilePath = newFilePath.replace("/", "@@");
          File newFile = new File(newFilePath);
          copyFile(oldFile, newFile);
        }
      }
    }

    fileNames = cropFiles.list();

    for (String fileName : fileNames) {
      LMODObject lmodObj = new LMODObject(fileName, "vegetations", "", "veg");
      lmodObj.data = readFile(cropsPath + "/" + fileName, "crop");
      if (lmodObj.path.isEmpty()) {
        lmodObj.path = "vegetations";
      }

      //System.out.println("Processing " + fileName);
      if (NameExistsInDB(lmodObj)) {
        if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
        {
          updateRecord(lmodObj);
        }

        if (DataIsIdentical(lmodObj)) {
          System.out.println("Skip Record, data is identical! " + fileName);
        } else //Data exists in the db, but it's different from our records.
        {
          updateRecord(lmodObj);
        }
      } else {
        //This will be a new record, not yet in LMOD Insert
        //System.out.println("Insert Record: " + lmodObj.path +"\\"+ lmodObj.name);
        insertRecord(lmodObj);
      }
    }

    //Delete all the temporary files
    //File cPath = new File(cropsPath);
    //deleteDirectory(cPath);
  }


  public void createOperationFiles() throws SQLException, FileNotFoundException, IOException, ServiceException {
    //Create duplicate files for records in the translation file
    File opFiles = new File(operationsPath);
    String[] fileNames = opFiles.list();

    //Create duplicate files for records in the translation file
    for (int i = 0; i < fileNames.length; ++i) {
      String oldName = fileNames[i].substring(0, fileNames[i].length() - 5);
      WEPSTranslationEntry e;

      if (translationExists(oldName)) {
        e = (WEPSTranslationEntry) translations.get(translationIndex(oldName));

        String oldFilePath = operationsPath + "\\" + e.name + ".oprn";
        File oldFile = new File(oldFilePath);
        ArrayList aliases = e.getAliases();

        for (Object o : aliases) {
          String name = (String) o;
          String newFilePath = operationsPath + "\\" + name + ".oprn";
          newFilePath = newFilePath.replace("/", "@@");
          File newFile = new File(newFilePath);
          copyFile(oldFile, newFile);
        }
      }
    }

    fileNames = opFiles.list();

    for (String fileName : fileNames) {
      LMODObject lmodObj = new LMODObject(fileName, "operations", "", "ope");
      lmodObj.data = readFile(operationsPath + "/" + fileName, "oprn");
      if (lmodObj.path.isEmpty()) {
        lmodObj.path = "operations";
      }

      //System.out.println("Processing " + fileName);
      if (NameExistsInDB(lmodObj)) {
        if (!AlreadyHasData(lmodObj))//Name exists, but no data, update the record with the data.
                    ;//updateRecord(lmodObj);
        else //Some data already exists in the db for this filename
        if (DataIsIdentical(lmodObj)) {
          System.out.println("Skip Record, data is identical! " + fileName);
        } else {
          //Data exists in the db, but it's different from our records.
          //System.out.println("Update Record: " + fileName);
          //updateRecord(lmodObj);
        }
      } else {
        //This will be a new record, not yet in LMOD Insert
        insertRecord(lmodObj);
        System.out.println("Insert New Record: " + fileName);
      }
    }

    //Delete all the temporary files
    //File opsPath = new File(operationsPath);
    //deleteDirectory(opsPath);
  }


  private boolean NameExistsInDB(LMODObject lmodObj) throws SQLException, ServiceException {
    boolean found;
    try (
        Statement statement = connection.createStatement();) {

      String query = "SELECT file_name FROM file_metadata"
          + " WHERE file_name = '" + lmodObj.name
          + "' AND file_path = '" + lmodObj.path + "'";
      ResultSet resultSet = statement.executeQuery(query);
      found = false;
      int counter = 1;
      while (resultSet.next()) {
        if (counter > 1) {
          System.out.println("More than one matching record: " + lmodObj.path + lmodObj.name);
        }

        found = true;
        counter++;
      }
    }
    return found;
  }


  private boolean AlreadyHasData(LMODObject lmodObj) throws SQLException, ServiceException {
    boolean result;
    try (
        Statement statement = connection.createStatement();) {
      String query = "SELECT file_name FROM file_metadata"
          + " WHERE file_name = '" + lmodObj.name
          + "' AND wepsdata is not null"
          + " AND file_path = '" + lmodObj.path + "'";
      ResultSet resultSet = statement.executeQuery(query);
      result = resultSet.next();
    }
    return result;
  }


  private boolean DataIsIdentical(LMODObject lmodObj) throws SQLException, ServiceException {
    boolean result = false;

    try (
        Statement statement = connection.createStatement();) {
      String query = "SELECT wepsData FROM file_metadata"
          + " WHERE file_name = '" + lmodObj.name
          + "' AND file_path = '" + lmodObj.path + "'";

      ResultSet resultSet = statement.executeQuery(query);

      if (resultSet.next()) {
        String dbData = resultSet.getString(1);

        if (dbData.replaceAll("\\s+", "").equalsIgnoreCase(lmodObj.data.replaceAll("\\s+", ""))) {
          result = true;
        } else {
          System.out.println("Data is different: " + lmodObj.path + lmodObj.name);
        }
      }
    }
    return result;
  }


  /**
   *
   * @param filePath full path to file
   * @param extension file extension e.g. '.crop'
   * @return The full text of the file read into a string.
   * @throws FileNotFoundException
   */
  private String readFile(String filePath, String extension) throws FileNotFoundException, IOException {
    if (!filePath.contains("." + extension)) {
      filePath += "." + extension;
    }

    String wepsData = FileUtils.readFileToString(new File(filePath));

    return wepsData;
  }


  private boolean updateRecord(LMODObject lmodObj) throws SQLException, ServiceException {
    try (
        Statement statement = connection.createStatement();) {
      String query = "UPDATE file_metadata"
          + " SET wepsdata = $$" + lmodObj.data
          + "$$ WHERE file_name = $$" + lmodObj.name
          + "$$ AND file_path = $$" + lmodObj.path + "$$";

      System.out.println("Updating: " + lmodObj.path + "\\" + lmodObj.name);
      //boolean updateWorked = statement.executeUpdate(query) >= 1;

      //Log it to the stdout for now.
      //if(!updateWorked)
      //    System.out.println("Update Query Failed");
      //else
      //    System.out.println("Updated Record: " + lmodObj.path +"\\"+ lmodObj.name);
    }

    return true; //updateWorked;
  }


  private boolean insertRecord(LMODObject lmodObj) throws SQLException, ServiceException {
    fileKey = fileKey.add(BigInteger.ONE);
    try (Statement statement = connection.createStatement();) {
      Date now = new Date();

      String query = "INSERT INTO file_metadata (file_key, hash_key, hash_data, file_path, file_name, object_key, import_date, wepsdata) "
          + "VALUES (" + fileKey + ", "
          + 0 + ", "
          + 0 + ", '"
          + lmodObj.path + "', '"
          + lmodObj.name + "', \'"
          + lmodObj.objectKey + "\', \'"
          + now.toString() + "\', $$"
          + lmodObj.data + "$$)";

      System.out.println("Inserting: " + lmodObj.path + "\\" + lmodObj.name + " new filekey: " + fileKey.toString());

      boolean insertWorked = statement.executeUpdate(query) == 1;

      //Log it to the stdout for now.
      if (!insertWorked) {
        System.out.println("Update query failed - Query: " + query);
      } else {
        System.out.println("insertRecord: " + lmodObj.path + lmodObj.name);
      }
    }

    return false;
  }


  public void copyFile(File from, File to) throws IOException {
    if (!from.exists()) {
      System.out.println("Bad Translation File Entry - Source File doesn't exist:" + from.getName());
      return;
    }
    if (!to.exists()) {
      to.getParentFile().mkdirs();
      Files.copy(from.toPath(), to.toPath());
    }
  }


  public void unZip(String zipFile, String outputFolder) {
    byte[] buffer = new byte[1024];

    try {
      File folder = new File(outputFolder);
      if (!folder.exists()) {
        folder.mkdir();
      }

      ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile));
      ZipEntry ze = zis.getNextEntry();

      while (ze != null) {
        String fileName = ze.getName();
        File newFile = new File(outputFolder + File.separator + fileName);

        //create all non exists folders
        //else you will hit FileNotFoundException for compressed folder
        new File(newFile.getParent()).mkdirs();

        FileOutputStream fos = new FileOutputStream(newFile);

        int len;
        while ((len = zis.read(buffer)) > 0) {
          fos.write(buffer, 0, len);
        }

        fos.close();
        ze = zis.getNextEntry();
      }

      zis.closeEntry();
      zis.close();
    } catch (IOException ex) {
      ex.printStackTrace();
    }
  }


  public boolean deleteDirectory(File path) {
    if (!path.exists()) {
      System.out.println("Cannot find directory that doesn't exist.");
      return false;
    }
    boolean ret = true;
    if (path.isDirectory()) {
      for (File f : path.listFiles()) {
        ret = ret && deleteDirectory(f);
        if (ret == false) {
          System.out.println("Cannot delete file: " + f.getName());
        }
      }
    }
    return ret && path.delete();
  }


  public void readTrans() throws FileNotFoundException, IOException {
    translations = new ArrayList();
    BufferedReader br = new BufferedReader(new FileReader(transPath));
    String curLine = br.readLine();
    ArrayList name = new ArrayList();
    ArrayList action = new ArrayList();
    ArrayList newname = new ArrayList();
    String[] temp;

    while (curLine != null) {
      if (curLine.isEmpty()) {
        curLine = br.readLine();
        continue;
      }
      if (curLine.charAt(0) != '#') {
        temp = curLine.split("\\s*\\|\\s*");
        if (temp[1].contains("rename")) {
          if (!translationExists(temp[2])) {
            WEPSTranslationEntry entry = new WEPSTranslationEntry(temp[2]);
            entry.addAlias(temp[0].replace("\\", "~~"));
            translations.add(entry);
          } else {
            int idx = translationIndex(temp[2]);
            WEPSTranslationEntry entry = (WEPSTranslationEntry) translations.get(idx);
            entry.addAlias(temp[0].replace("\\", "~~"));
          }
        }
      }

      curLine = br.readLine();
    }
  }


  public boolean translationExists(String name) {
    try {
      for (Object o : translations) {
        WEPSTranslationEntry e = (WEPSTranslationEntry) o;
        if (e.name.equals(name)) {
          return true;
        }
      }
    } catch (Exception e) {
      //who really cares, you failed to give me a string.
    }
    return false;
  }


  public int translationIndex(String name) {
    try {
      for (int i = 0; i < translations.size(); ++i) {
        WEPSTranslationEntry e = (WEPSTranslationEntry) translations.get(i);
        if (e.name.equals(name)) {
          return i;
        }
      }
    } catch (Exception e) {
      //who really cares, you failed to give me a string.
    }
    return -1;
  }

  private class LMODObject {

    public String path;
    public String name;
    public String data;
    public String objectKey;


    public LMODObject() {
    }


    public LMODObject(String filename, String basePath) {
      if (filename.contains("~~")) {
        this.path = basePath + "\\" + filename.substring(0, filename.indexOf("~~"));
        //this.path = this.path.toLowerCase();
        filename = filename.substring(filename.indexOf("~~") + 2, filename.length());
      } else {
        this.path = "";
      }
      if (filename.contains(".oprn") || filename.contains(".crop")) {
        filename = filename.substring(0, filename.lastIndexOf("."));
      }
      filename = filename.replace("@@", "/");
      //filename = filename.toLowerCase();
      this.name = filename;
    }


    public LMODObject(String filename, String basePath, String data, String objectKey) {
      if (filename.contains("~~")) {
        this.path = basePath + "\\" + filename.substring(0, filename.indexOf("~~"));
        //this.path = this.path.toLowerCase();
        filename = filename.substring(filename.indexOf("~~") + 2, filename.length());
      } else {
        this.path = basePath;
      }
      if (filename.contains(".oprn") || filename.contains(".crop")) {
        filename = filename.substring(0, filename.lastIndexOf("."));
      }

      filename = filename.replace("@@", "/");
      //filename = filename.toLowerCase();
      this.name = filename;
      this.data = data;
      this.objectKey = objectKey;
    }
  }
}