ServiceTest.java [src/csip/test] Revision: Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.test;
import csip.utils.Client;
import csip.utils.JSONUtils;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.Reader;
import java.net.URL;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.logging.StreamHandler;
import org.apache.commons.io.FileUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.skyscreamer.jsonassert.JSONAssert;
/**
* Service Testing.
*
* @author odavid
*/
public class ServiceTest {
static final String KEYSONLY = "keysonly";
static final String KEYVALUE = "keyvalue";
static final String KEYVALUEORDER = "keyvalueorder";
static final String NONE = "none";
static final int NO_WARMUP = 0;
static final int WARMUP_AND_RUN = 1;
static final int WARMUP_ONLY = 2;
static final List<String> VALIDTESTING = Arrays.asList(KEYSONLY, KEYVALUE, KEYVALUEORDER, NONE);
File folder;
File[] files;
String url;
String testing;
int concurrent;
int workers;
int timeout;
int modelTimeout;
int delay;
int limit;
int warmup;
int warmupRuns;
boolean downloadFiles;
String logLevel = "WARNING";
String config;
boolean ignore = false;
static final Logger log = Logger.getLogger("ServiceTest");
static {
log.setLevel(Level.SEVERE);
SimpleFormatter fmt = new SimpleFormatter();
StreamHandler sh = new StreamHandler(System.out, fmt);
log.addHandler(sh);
}
private ServiceTest(File target) throws Exception {
if (!target.exists()) {
throw new IllegalArgumentException("Not a folder or file: " + target);
}
if (target.getName().endsWith("req.json") && target.isFile()) {
folder = target.getParentFile();
files = new File[]{target};
} else if (target.isDirectory()) {
folder = target;
files = folder.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.getName().endsWith("-req.json");
}
});
// sort the file names.
Arrays.sort(files, new Comparator<File>() {
@Override
public int compare(File o1, File o2) {
int n1 = extractNumber(o1.getName());
int n2 = extractNumber(o2.getName());
if ((n1 == 0) || (n2 == 0)) {
return o1.compareTo(o2);
} else {
return n1 - n2;
}
}
// to do - could make char s and char e configurable
// and this would enable custom ways to extract a number
// from the file name for sorting
private int extractNumber(String name) {
int i = 0;
try {
int s = name.lastIndexOf('t') + 1;
int e = name.lastIndexOf('-');
if ((s < 0) || (e < 0)) {
s = name.lastIndexOf('_') + 1;
e = name.lastIndexOf('.');
}
String number = name.substring(s, e);
i = Integer.parseInt(number);
} catch (NumberFormatException nfe) {
i = 0;
}
return i;
}
});
} else {
throw new IllegalArgumentException("Invalid argument: " + target);
}
// load the service testing settings.
Properties p = loadProperties(folder);
testing = p.getProperty("verify", NONE); // strict, keys
if (!VALIDTESTING.contains(testing)) {
throw new RuntimeException("invalid testing method: " + testing);
}
ignore = Boolean.parseBoolean(p.getProperty("ignore", "false"));
url = p.getProperty("url");
if (url == null) {
String host = p.getProperty("host");
String path = p.getProperty("path");
if (host == null && path == null) {
throw new RuntimeException("'url' (or host/path) not found in " + new File(folder, "service.properties"));
}
url = host + path;
}
concurrent = Integer.parseInt(p.getProperty("concurrency", "1"));
if (concurrent < 1) {
concurrent = 1;
}
timeout = Integer.parseInt(p.getProperty("timeout", "3600"));
if (timeout < 1000) {
timeout = 1000;
}
modelTimeout = Integer.parseInt(p.getProperty("modelTimeout", "600000")); // default is 10 minutes
if (modelTimeout <= 0) {
modelTimeout = 600000;
}
delay = Integer.parseInt(p.getProperty("delay", "0"));
if (delay < 0) {
delay = 0;
}
workers = Integer.parseInt(p.getProperty("workers", "1"));
if (workers < 0) {
workers = 1;
}
// warmup phase: 0-off, 1-on
warmup = Integer.parseInt(p.getProperty("warmup", "0"));
if (warmup < 0) {
warmup = NO_WARMUP;
}
warmupRuns = Integer.parseInt(p.getProperty("warmupRuns", "4"));
if (warmupRuns < 0) {
warmupRuns = 4;
}
downloadFiles = Boolean.parseBoolean(p.getProperty("download_files", "true"));
logLevel = p.getProperty("logLevel", "WARNING");
if ((logLevel != null) && (logLevel.length() > 0)) {
log.setLevel(Level.parse(logLevel));
}
config = p.getProperty("config", "service-conf.json");
limit = Integer.parseInt(p.getProperty("limit", "-1"));
limit = (limit < 1) ? files.length : limit;
}
/**
* Execute the test.
*
* @return
*/
private Results test() {
final Results results = new Results();
if (ignore) {
results.incIgnored();
return results;
}
final Client client = new Client(timeout, log);
// apply configuration
if (config != null) {
// changed with absolute name
File conf = new File(config);
if (!conf.exists()) {
// relative name
conf = new File(folder, config);
}
if (conf.exists()) {
try {
JSONObject o = new JSONObject(FileUtils.readFileToString(conf, "UTF-8"));
client.doPUT(getBaseURLContext(url) + "/c/conf", o);
} catch (Exception E) {
log.warning("Failed to load config: " + conf);
}
} else {
// if there is no config... then assume service is pre-configured...
}
}
int maxFiles = limit;
final CountDownLatch latch = new CountDownLatch(maxFiles);
final CountDownLatch warmupLatch = new CountDownLatch(workers * warmupRuns);
final ConcurrentLinkedQueue<File> fileQueue = new ConcurrentLinkedQueue<>();
final ConcurrentLinkedQueue<File> warmups = new ConcurrentLinkedQueue<>();
for (int i = 0; i < maxFiles; i++) {
if ((i < workers * warmupRuns) && (warmup != NO_WARMUP)) {
warmups.add(files[i]);
}
fileQueue.add(files[i]);
}
ExecutorService warmupExecutor = null;
final ExecutorService executor = Executors.newFixedThreadPool(concurrent, new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "exeuctor-thread");
}
});
// final ExecutorService executor = new ThreadPoolExecutor(
// concurrent, concurrent, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()) {
// @Override
// protected void afterExecute(Runnable r, Throwable t) {
// super.afterExecute(r, t);
// if (t != null) {
// System.out.println("Perform exception handler logic");
// }
// final File file = fileQueue.poll();
// if (file != null) {
// if (delay > 0) {
// try {
// System.out.println("THIS WORKER SLEEP FOR:" + delay);
// Thread.sleep(delay);
// } catch (InterruptedException ex) {
// System.out.println("Interruption during sleep - non-init!!!");
// }
// }
// // keep going
// submit(new Callable<Void>() {
// @Override
// public Void call() {
// return t(file, results, client, latch);
// }
// });
// }
// else
// {
// System.out.println("THE FILE IS NULL - THIS SHOULD NOT HAPPEN UNTIL END, filequeue-size=" + fileQueue.size());
// }
// }
// };
// submit the first run twice as a "warmup" across the workers
if (warmup != NO_WARMUP) {
final Results warmupResults = new Results();
warmupExecutor = Executors.newFixedThreadPool(workers * warmupRuns, new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "warmup-thread");
}
});
for (int i = 0; i < (workers * warmupRuns); i++) {
final File file = warmups.poll();
log.info("SUBMITTING WARM UP RUN=" + file);
warmupExecutor.submit(new Callable<Void>() {
@Override
public Void call() {
return t(file, warmupResults, client, warmupLatch);
}
});
}
}
// submit the initial runs.
if (warmup != WARMUP_ONLY) {
// for (int i = 0; i < Math.min(concurrent, maxFiles); i++) {
for (int i = 0; i < maxFiles; i++) {
if (delay > 0) {
try {
Thread.sleep(delay);
} catch (InterruptedException ex) {
log.warning("Interrupted during sleep- init runs!");
}
}
final File file = fileQueue.poll();
log.info("SUBMITTING TO PROCESS FILE=" + file);
executor.submit(new Callable<Void>() {
@Override
public Void call() {
return t(file, results, client, latch);
}
});
}
}
try {
if (warmup != WARMUP_ONLY) {
latch.await();
}
if (warmup != NO_WARMUP) {
warmupLatch.await();
}
} catch (InterruptedException ex) {
// whatever
}
executor.shutdown();
if (warmup != NO_WARMUP) {
warmupExecutor.shutdown();
}
return results;
}
/**
*
* @param reqFile
* @param results
* @param client
* @param latch
* @return
*/
private Void t(File reqFile, Results results, Client client, CountDownLatch latch) {
try {
if (delay > 0) {
try {
Thread.sleep(delay);
} catch (InterruptedException ex) {
log.warning("Interrupted during sleep: pre-run-sleep");
}
}
File res = getResFile(reqFile);
// check if there is a async result with the status "Running"
if (res.exists()) {
res.delete();
// // ASYNC PART
// // check if the prev run was running.
// JSONObject prev_json = new JSONObject(FileUtils.readFileToString(res));
// if (JSONUtils.getStatus(prev_json).equals("Finished")) {
// log.warning("Skipped (Finished)" + " " + reqFile + "... ");
// results.incSkipped();
// log.info("EXITING test run,,, latch count-down?");
// return null;
// }
//
// if (JSONUtils.getStatus(prev_json).equals("Running") || JSONUtils.getStatus(prev_json).equals("Submitted")) {
// // it is running, need to check again.
// String q_url = getBaseURLContext(url) + "/q/" + JSONUtils.getSID(prev_json);
//
// // get the new
// client.doGET(q_url, res);
//
// // check the status again
// JSONObject this_json = new JSONObject(FileUtils.readFileToString(res));
// String status = JSONUtils.getStatus(this_json);
// log.info(status + " " + reqFile + "... ");
// if (status.equals("Finished")) {
//
// // dowload
// download(this_json, getResFolder(reqFile), client);
//
// if (testing.equals(NONE)) {
// results.incSuccess();
// log.info("ASYNC EXIT NONE");
// return null;
// }
// // test
// runtest(reqFile, this_json);
// }
// results.incSuccess();
// log.info("ASYNC EXIT END");
// return null;
// }
}
JSONObject req_json = new JSONObject(FileUtils.readFileToString(reqFile, "UTF-8"));
boolean isAsync = JSONUtils.isAsync(req_json);
// input
File inp = getInpFolder(reqFile);
File[] attachments = (inp != null) ? inp.listFiles() : new File[0];
// delete output if exist.
FileUtils.deleteDirectory(getResFolderName(reqFile));
// run the service
JSONObject response = client.doPOST(url, req_json, attachments, (Map<String,String>) null);
FileUtils.writeStringToFile(getResFile(reqFile), response.toString(2).replace("\\/", "/"), "UTF-8");
// print the error if failed.
if (JSONUtils.getStatus(response).equals("Failed")) {
JSONObject respose_metainfo = response.getJSONObject("metainfo");
throw new Exception(respose_metainfo.getString("error"));
}
if (isAsync) {
results.incSuccess();
log.info("THREAD EXECUTOR TEST SUCCESS:" + reqFile);
return null;
}
if (!JSONUtils.hasResult(response)) {
throw new Exception("No 'result' in response");
}
if (downloadFiles) {
download(response, getResFolder(reqFile), client);
}
if (testing.equals(NONE)) {
results.incSuccess();
log.info("EXITING test run,,, latch count-down?");
return null;
}
runtest(reqFile, response);
results.incSuccess();
} catch (Exception | Error E) {
System.out.println(" Failed: " + reqFile);
log.warning("ERROR-" + E.toString());
for (StackTraceElement ste : E.getStackTrace()) {
log.warning(ste.getClassName() + ":" + ste.getMethodName() + ":" + ste.getLineNumber() + "---" + ste.toString());
}
System.out.println(E.getMessage());
results.incFailed();
} finally {
log.info("THREAD EXECUTOR COUNT DOWN AFTER:" + reqFile);
latch.countDown();
}
return null;
}
private void runtest(File reqFile, JSONObject response) throws Exception {
JSONArray response_result = response.getJSONArray("result");
// all real checking starts here.
// golden response
JSONObject golJson = new JSONObject(FileUtils.readFileToString(getResponse(reqFile), "UTF-8"));
// check output files
JSONArray response_golden = golJson.getJSONArray("result");
Map<String, JSONObject> rmap = JSONUtils.preprocess(response_result);
Map<String, JSONObject> golmap = JSONUtils.preprocess(response_golden);
for (String key : rmap.keySet()) {
JSONObject val = rmap.get(key);
String url = val.getString("value");
if (url.startsWith("http") && url.endsWith(key)) {
response_result.remove(val);
golmap.remove(golmap.get(key));
}
}
File out = getRefFolder(reqFile);
if (out != null) {
File[] outFiles = out.listFiles();
for (File file : outFiles) {
String f = file.getName();
if (!rmap.containsKey(f)) {
throw new Exception("Expecting file in result: " + f);
}
// TODO compare the output files.
}
}
switch (testing) {
case KEYVALUE:
compareStrict(response_result, response_golden);
break;
case KEYSONLY:
compareKeys(response_result, response_golden);
break;
default:
throw new IllegalArgumentException(testing);
}
}
/**
*
*/
public static class Results {
int succeeded = 0;
int failed = 0;
int skipped = 0;
int ignored = 0;
void add(Results other) {
succeeded += other.getSucceeded();
failed += other.getFailed();
skipped += other.getSkipped();
ignored += other.getIgnored();
}
public int getTotal() {
return succeeded + failed + skipped + ignored;
}
public int getFailed() {
return failed;
}
public int getSkipped() {
return skipped;
}
public int getSucceeded() {
return succeeded;
}
public int getIgnored() {
return ignored;
}
synchronized void incSuccess() {
succeeded++;
}
synchronized void incFailed() {
failed++;
}
synchronized void incSkipped() {
skipped++;
}
synchronized void incIgnored() {
ignored++;
}
@Override
public String toString() {
return "=================" + "\n"
+ " Total: " + getTotal() + "\n"
+ " Failed: " + getFailed() + "\n"
+ " Succeeded: " + getSucceeded() + "\n"
+ " Skipped: " + getSkipped() + "\n"
+ " Ignored: " + getIgnored() + "\n";
}
}
static void download(JSONObject resp, File resultFolder, Client client) throws Exception {
JSONArray arr = resp.getJSONArray("result");
// download files using the 'q' service.
Map<String, JSONObject> rmap = JSONUtils.preprocess(arr);
for (String key : rmap.keySet()) {
JSONObject val = rmap.get(key);
String url = val.getString("value");
if (url == null) {
throw new Exception("Expecting file in result: " + key);
}
if (url.startsWith("http") && url.contains("/q/") && url.endsWith(key)) {
if (!resultFolder.exists()) {
resultFolder.mkdirs();
}
File outFile = new File(resultFolder, key);
client.doGET(url, outFile);
if (!outFile.exists()) {
throw new Exception("Missing output file: " + key);
}
}
}
}
static File getInpFolder(File request) {
return getFolder(request, "-req");
}
static File getRefFolder(File request) {
return getFolder(request, "-ref");
}
static File getResFolderName(File request) {
String name = request.getPath();
return new File(name.replace("-req.json", "-res"));
}
static File getResponse(File request) {
String name = request.getPath();
if (!name.endsWith("-req.json")) {
return null;
}
File response = new File(name.replace("-req.json", "-res.json"));
if (!response.exists() || !request.canRead() || !response.canRead()) {
return null;
}
return response;
}
static File getResFile(File request) {
String name = request.getPath();
File res = new File(name.replace("-req.json", "-res.json"));
return res;
}
static File getResFolder(File request) {
String name = request.getPath();
File res = new File(name.replace("-req.json", "-res"));
return res;
}
static File getFolder(File request, String postfix) {
String name = request.getPath();
if (!name.endsWith("-req.json")) {
return null;
}
File inp = new File(name.replace("-req.json", postfix));
if (!inp.exists() || !inp.canRead() || !inp.isDirectory()) {
return null;
}
return inp;
}
static void compareKeys(JSONArray res, JSONArray gol) throws Exception {
Map<String, JSONObject> preq = JSONUtils.preprocess(res);
Map<String, JSONObject> pgol = JSONUtils.preprocess(gol);
for (String key : pgol.keySet()) {
if (!preq.containsKey(key)) {
throw new Exception("Missing key in result:" + key);
}
preq.remove(key);
}
if (!preq.keySet().isEmpty()) {
throw new Exception("Extra key(s) in result:" + preq.keySet().toString());
}
}
static void compareStrict(JSONArray res, JSONArray gol) throws Exception {
JSONAssert.assertEquals(gol.toString(), res.toString(), false);
}
static Properties loadProperties(File folder) throws Exception {
Properties p = new Properties();
File props = new File(folder.getParentFile(), "service.properties");
if (props.exists()) {
Reader is = new FileReader(props);
p.load(is);
is.close();
}
props = new File(folder, "service.properties");
if (!props.exists()) {
throw new RuntimeException("Not found: " + props);
}
Reader is = new FileReader(props);
p.load(is);
is.close();
return p;
}
static String getBaseURLContext(String url) {
// this is the first part (host + port)
String host = url.substring(0, url.indexOf("/", url.indexOf("://") + 3));
// this is the second part (context)
String context = url.substring(host.length() + 1, url.indexOf("/", host.length() + 1));
return host + "/" + context;
}
public static Results run(String target) throws Exception {
return new ServiceTest(new File(target)).test();
}
public static Results run(String[] targets) throws Exception {
Results summary = new Results();
for (String target : targets) {
Results result = new ServiceTest(new File(target)).test();
summary.add(result);
}
return summary;
}
public static void main(String[] args) throws Exception {
System.out.println(run(args));
}
}