GetWEPSArchives.java [tools/GetWEPSArchives/src/getwepsarchives] Revision: 04954fb50d3148fbace9e091840b77662ab98631  Date: Fri Nov 22 12:54:54 MST 2019
/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package getwepsarchives;

import archives.FileArchive;
import archives.MongoArchive;
import archives.ServiceArchive;
import csip.ServiceException;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Level;
import java.util.logging.Logger;
import models.ModelArchive;
import models.WEPSModelArchive.WEPSMetaData;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;

/**
 *
 * @author <a href="mailto:shaun.case@colostate.edu">Shaun Case</a>
 */
public class GetWEPSArchives {

    static FileOutputStream outFile;
    static BufferedWriter fileWriter;

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) throws SQLException, Exception {
	ServiceArchive mongoArchive;
	String dbLocation;
	ArrayList<ModelArchive> models = null;

	if (args.length >= 2) {
	    ArrayList<ModelArchive> results = null;

	    dbLocation = args[0];
	    if (dbLocation.contains("mongodb://")) {
		mongoArchive = new MongoArchive(dbLocation);
	    } else {
		mongoArchive = new FileArchive(dbLocation);
	    }

	    models = mongoArchive.getArchivesByFilter("service", args[1], -1);

	    if (null != models) {
		for (ModelArchive model : models) {
		    JSONObject input = model.getOriginalRequest();
		    JSONObject result = model.getOriginalResponse();

		}
	    }

	    try {
		mongoArchive.shutdown();
	    } catch (Exception ex) {
		Logger.getLogger(GetWEPSArchives.class.getName()).log(Level.SEVERE, null, ex);
	    }

	} else {
	    if (args.length == 1) {
		if (args[0].contains(".txt")) {
		    // Then we have a list of suids to fetch.
		    ConcurrentHashMap<String, WEPSMetaData> metaDataResults = new ConcurrentHashMap<>();

		    boolean fileDone = false;
		    int lineNumber = 0;
		    int maxThreads = 15;

		    File inputFile = new File(args[0]);
		    BufferedReader inputStream = Files.newBufferedReader(inputFile.toPath());

		    dbLocation = inputStream.readLine();

		    if (null == dbLocation) {
			printCommandLineHelp();
			throw new Exception("No mongo archive db Location found in first line of text file.");
		    }

		    String[] params = dbLocation.split(",");

		    dbLocation = params[0];
		    if (params.length > 1) {
			maxThreads = Integer.parseInt(params[1].trim());
		    }

		    writeOutputResultHeaders(args[0]);

		    while (!fileDone) {
			int count = 0;
			String[] suid = new String[maxThreads];

			while ((count < maxThreads) && (!fileDone)) {
			    String inLine = inputStream.readLine();

			    if (null != inLine) {
				lineNumber++;
				suid[count] = inLine.trim();
				count++;
			    } else {
				fileDone = true;
			    }
			}

			CountDownLatch latch = new CountDownLatch(count);
			for (int i = 0; i < count; i++) {
			    ArchiveThread archiveThread = new ArchiveThread(suid[i], dbLocation, metaDataResults, latch);
			    archiveThread.start();
			}

			latch.await();
			writeOutputResults(metaDataResults);
			metaDataResults.clear();
		    }
		    fileWriter.close();
		    outFile.close();
		} else {
		    printCommandLineHelp();
		}
	    } else {
		printCommandLineHelp();
	    }

	}
    }

    protected static void writeOutputResultHeaders(String fileName) throws FileNotFoundException, IOException {

	outFile = new FileOutputStream(fileName + ".csv");
	fileWriter = new BufferedWriter(new OutputStreamWriter(outFile));

	fileWriter.write("suid, cokey, latitude, longitude, annualPrecip, windEnergy, component_name, surface_sand, surface_silt, surface_clay, albedo, slope_gradient, surface_thickness, num_layers, aggregateStability, crustStability, soilWiltPoint, errosionRate" + System.lineSeparator());
	fileWriter.flush();

    }

    protected static void writeOutputResults(ConcurrentHashMap<String, WEPSMetaData> metaDataResults) throws FileNotFoundException, IOException {
	if ((null != metaDataResults) && !metaDataResults.isEmpty()) {

	    for (String key : metaDataResults.keySet()) {
		WEPSMetaData wepsMetaData = metaDataResults.get(key);

		fileWriter.write(
			wepsMetaData.suid() + ", " 
			+ wepsMetaData.cokey() + ", " 
			+ wepsMetaData.latitude() + ", " 
			+ wepsMetaData.longitude() + ", " 				
			+ wepsMetaData.annualPrecip()
			+ ", " + wepsMetaData.windEnergy() + ", "
			+ "\"" + wepsMetaData.componentName() + "\", "
			+ wepsMetaData.fractionSand(0) + ", "
			+ wepsMetaData.fractionSilt(0) + ", "
			+ wepsMetaData.fractionClay(0) + ", "
			+ wepsMetaData.albedo() + ", "
			+ wepsMetaData.slope_gradient() + ", "
			+ wepsMetaData.surface_thickness() + ", "
			+ wepsMetaData.num_layers() + ", "
			+ wepsMetaData.aggStability() + ", " 
			+ wepsMetaData.crustStability() + ", "
			+ wepsMetaData.soilWiltPoint() + ", " 
			+ wepsMetaData.errosionRate());
		fileWriter.write(System.lineSeparator());
		fileWriter.flush();
		if (!wepsMetaData.errorMessages().isEmpty()) {
		    System.err.println(System.lineSeparator() + wepsMetaData.suid() + ":" + System.lineSeparator() + wepsMetaData.errorMessages());
		}
	    }
	    fileWriter.flush();
	    outFile.flush();
	}
    }

    protected static void printCommandLineHelp() {
	System.out.println("ERROR: No input arguments specified.\nPlease specify the URI location of the Archive to search and either the IP address to search for or a filename of a file containing a list of SUID's to search for.  The filename must end in '.txt'");
	System.out.println("Usage:");
	System.out.println("\tCokeyConverter <Archive URI> <[ip address] | [filename]> <new service URI>");
	System.out.println("\t\tWhere <Archive URI> is a valid host name port number combination.");
	System.out.println("\t\tAnd [ip address] is a valid IPv4 address");
	System.out.println("\t\tOr [filename] points to a file that contains a list of SUID's, one per line, to search for.");
	System.out.println("\t\t\tAdditionally, a file may contain only one SUID, and if so, then the search will be for all records containing the ip address used in the SUID specified, and found after the date of the SUID found, for the service specified for that SUID.");
    }

    public static Map<String, JSONObject> preprocess(JSONArray params) throws JSONException {
	Map<String, JSONObject> p = new HashMap<>();
	for (int i = 0; i < params.length(); i++) {
	    JSONObject o = params.getJSONObject(i);
	    p.put(o.getString("name"), o);
	}
	return p;
    }
}