ServiceTest2.java [src/csip/test] Revision: Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.test;
import csip.utils.Client;
import csip.ModelDataService;
import csip.utils.JSONUtils;
import csip.utils.Services;
import java.io.File;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.logging.StreamHandler;
import org.apache.commons.io.FileUtils;
import org.apache.http.client.utils.URIBuilder;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.skyscreamer.jsonassert.JSONAssert;
/**
* Service Testing.
*
* @author odavid
*/
public class ServiceTest2 {
static final String KEYSONLY = "keysonly";
static final String STRICT = "strict";
static final String KEYVALUE = "keyvalue";
static final String NONE = "none";
static final List<String> VALIDTESTING = Arrays.asList(KEYSONLY, KEYVALUE, STRICT, NONE);
File folder;
File[] files;
File target;
String testing = NONE;
int concurrent = 1;
int timeout = 1000;
int delay = 0;
int rounds = 1;
String contextUrl = null;
boolean downloadFiles = true;
final JSONArray reports = new JSONArray();
static final Logger log = Logger.getLogger("ServiceTest");
// static final StreamHandler sh = new StreamHandler(System.out, new SimpleFormatter());
static final int _100KB = 102400;
static final int _64KB = 64 * 1024;
static final int LIMIT = _64KB;
static {
// sh.setLevel(Level.OFF);
log.setLevel(Level.OFF);
// log.addHandler(sh);
log.setUseParentHandlers(false);
}
private ServiceTest2(File target, Properties config) throws Exception {
if (!target.exists()) {
throw new IllegalArgumentException("Not a folder or file: " + target);
}
this.target = target;
if (target.getName().endsWith(".json") && target.isFile()) {
folder = target.getParentFile();
files = new File[]{target};
} else if (target.isDirectory()) {
folder = target;
files = folder.listFiles((File pathname) -> pathname.getName().endsWith(".json"));
Arrays.sort(files);
} else {
throw new IllegalArgumentException("Invalid argument: " + target);
}
// load the service testing settings.
String s = config.getProperty("csip.test.verify", System.getProperty("csip.test.verify"));
if (VALIDTESTING.contains(s)) {
testing = s;
}
s = config.getProperty("csip.test.contexturl", System.getProperty("csip.test.contexturl"));
if (s != null && !s.isEmpty()) {
contextUrl = s;
}
s = config.getProperty("csip.test.concurrency", System.getProperty("csip.test.concurrency"));
if (s != null) {
try {
concurrent = Integer.parseInt(s);
} catch (NumberFormatException E) {
}
}
s = config.getProperty("csip.test.timeout", System.getProperty("csip.test.timeout"));
if (s != null) {
try {
timeout = Integer.parseInt(s);
} catch (NumberFormatException E) {
}
}
s = config.getProperty("csip.test.delay", System.getProperty("csip.test.delay"));
if (s != null) {
try {
delay = Integer.parseInt(s);
} catch (NumberFormatException E) {
}
}
s = config.getProperty("csip.test.rounds", System.getProperty("csip.test.rounds"));
if (s != null) {
try {
rounds = Integer.parseInt(s);
} catch (NumberFormatException E) {
}
}
// warmup phase: 0-off, 1-on
s = config.getProperty("csip.test.downloadfiles", System.getProperty("csip.test.downloadfiles"));
if (s != null) {
downloadFiles = Boolean.parseBoolean(s);
}
s = config.getProperty("csip.test.loglevel", System.getProperty("csip.test.loglevel"));
if (s != null && !s.isEmpty()) {
// sh.setLevel(Level.parse(s));
log.setLevel(Level.parse(s));
}
}
/**
* Execute the test.
*
* @return
*/
JSONObject test() {
int attempts = 4;
int bq_size = 4;
final Stats stats = new Stats();
for (int i = 0; i < rounds; i++) {
Services.runParallel(files.length, concurrent, attempts, bq_size, new Services.CallableFactory() {
@Override
public Callable<?> create(final int i) {
return (Callable<?>) () -> {
if (delay > 0) {
try {
Thread.sleep(delay);
} catch (InterruptedException ex) {
}
}
JSONObject report = new JSONObject();
JSONObject result = null;
try {
File f = files[i];
JSONObject json = new JSONObject(FileUtils.readFileToString(f, "UTF-8"));
String url = JSONUtils.getMetaInfo(json).getString("service_url");
if (contextUrl != null) {
url = contextUrl + getServicePath(url);
}
System.out.println(f.getName() + " >>>>>>>>> " + url + " ");
System.out.flush();
report.put("file", f.toString());
report.put("service", url);
long p = Client.ping(url, timeout);
if (p == -1) {
throw new Exception("Url not reachable: " + url);
}
JSONObject req = JSONUtils.newRequest();
JSONObject req_meta = new JSONObject();
if (JSONUtils.getMetaInfo(json).has("request-results")) {
JSONArray a = JSONUtils.getMetaInfo(json).getJSONArray("request-results");
for (int j = 0; j < a.length(); j++) {
req_meta.accumulate("request-results", a.getString(j));
}
}
req.put("parameter", json.getJSONArray("parameter"));
req.put("metainfo", req_meta);
// System.out.println("REQ>>>>>>>>>>>>>>>>>>>>>> " + req.toString(4));
Client client = new Client(timeout, log);
JSONObject response = client.doPOST(url, req);
String resp = response.toString(2);
if (resp.length() > LIMIT) {
System.out.println("RESPONSE >>>>>>>>>>>>>>>>>>>>>> \n" + resp.substring(0, LIMIT) + " \n .... truncated, > 100KB");
} else {
System.out.println("RESPONSE >>>>>>>>>>>>>>>>>>>>>> \n" + resp);
}
if (response.has("metainfo")) {
report.put("metainfo", response.getJSONObject("metainfo"));
}
if (JSONUtils.getStatus(response).equals("Failed")) {
JSONObject respose_metainfo = response.getJSONObject("metainfo");
throw new Exception("Failed, " + respose_metainfo.getString("error"));
}
if (!JSONUtils.hasResult(response)) {
throw new Exception("No 'result' in response.");
}
// if (downloadFiles) {
// download(res, getResFolder(reqFile), client);
// }
//
result = runtest(json, response);
report.put("test", result);
stats.incSuccess();
} catch (Exception E) {
try {
report.put("test", E.getMessage());
stats.incFailed();
} catch (JSONException ex) {
log.log(Level.SEVERE, null, ex);
}
} finally {
synchronized (reports) {
reports.put(report);
System.out.println("SUMMARY >>>>>>>>>>>>>>>>>>>>>> " + report.toString(2));
System.out.println();
}
}
return null;
};
}
});
}
JSONObject r = new JSONObject();
try {
r.put("target", target.toString());
r.put("date", new Date());
r.put("total", stats.getTotal());
r.put("successful", stats.getSucceeded());
r.put("failed", stats.getFailed());
r.put("tests", reports);
} catch (JSONException ex) {
ex.printStackTrace();
}
return r;
}
/**
*
*/
static class Stats {
int succeeded;
int failed;
void add(Stats other) {
succeeded += other.getSucceeded();
failed += other.getFailed();
}
public int getTotal() {
return succeeded + failed;
}
public int getFailed() {
return failed;
}
public int getSucceeded() {
return succeeded;
}
synchronized void incSuccess() {
succeeded++;
}
synchronized void incFailed() {
failed++;
}
}
JSONObject runtest(JSONObject golden, JSONObject response) throws Exception {
JSONArray response_result = response.getJSONArray("result");
JSONArray response_golden = golden.getJSONArray("result");
Map<String, JSONObject> rmap = JSONUtils.preprocess(response_result);
Map<String, JSONObject> golmap = JSONUtils.preprocess(response_golden);
for (String key : rmap.keySet()) {
JSONObject val = rmap.get(key);
String url = val.getString("value");
if (url.startsWith("http") && url.endsWith(key)) {
response_result.remove(golmap.get(key));
response_golden.remove(golmap.get(key));
}
}
JSONObject o = new JSONObject();
o.put("testing", testing);
switch (testing) {
case KEYVALUE:
o.put("result", compareKeyValues(response_result, response_golden));
break;
case STRICT:
o.put("result", compareStrict(response_result, response_golden));
break;
case KEYSONLY:
o.put("result", compareKeys(response_result, response_golden));
break;
case NONE:
o.put("result", "no comparison tests.");
break;
default:
o.put("result", "unknown testing config: " + testing);
}
return o;
}
// static void download(JSONObject resp, File resultFolder, Client client) throws Exception {
// JSONArray arr = resp.getJSONArray("reports");
// // download files using the 'q' service.
// Map<String, JSONObject> rmap = JSONUtils.preprocess(arr);
// for (String key : rmap.keySet()) {
// JSONObject val = rmap.get(key);
// String url = val.getString("value");
// if (url == null) {
// throw new Exception("Expecting file in reports: " + key);
// }
// if (url.startsWith("http") && url.contains("/q/") && url.endsWith(key)) {
// if (!resultFolder.exists()) {
// resultFolder.mkdirs();
// }
// File outFile = new File(resultFolder, key);
// client.doGET(url, outFile);
// if (!outFile.exists()) {
// throw new Exception("Missing output file: " + key);
// }
// }
// }
// }
static JSONObject compareKeyValues(JSONArray res, JSONArray golden) throws Exception {
Map<String, JSONObject> pres = JSONUtils.preprocess(res);
Map<String, JSONObject> pgol = JSONUtils.preprocess(golden);
JSONObject r = new JSONObject();
for (String name : pgol.keySet()) {
if (!pres.containsKey(name)) {
r.accumulate("missing", name);
continue;
}
String resval = pres.get(name).get(ModelDataService.KEY_VALUE).toString();
String golval = pgol.get(name).get("value").toString();
if (resval.startsWith("http:") || resval.startsWith("https:") || resval.startsWith("file:")) {
r.accumulate("ignored", name);
continue;
}
try {
double dgolval = Double.parseDouble(resval);
double dresval = Double.parseDouble(golval);
if (dgolval != dresval) {
r.accumulate("differ", "'" + name + "': " + dgolval + " != " + dresval + " \n");
} else {
r.accumulate("ok", name);
}
} catch (NumberFormatException E) {
if (!golval.equals(resval)) {
r.accumulate("differ", "'" + name + "': '" + golval + "' != '" + resval + "' \n");
} else {
r.accumulate("ok", name);
}
}
}
return r;
}
static JSONObject compareKeys(JSONArray res, JSONArray golden) throws Exception {
JSONObject r = new JSONObject();
Map<String, JSONObject> pres = JSONUtils.preprocess(res);
Map<String, JSONObject> pgol = JSONUtils.preprocess(golden);
for (String key : pgol.keySet()) {
if (!pres.containsKey(key)) {
r.append("missing", key);
}
pres.remove(key);
}
if (!pres.keySet().isEmpty()) {
r.append("extra", pres.keySet().toString());
}
return r;
}
static JSONObject compareStrict(JSONArray res, JSONArray gol) throws Exception {
JSONObject o = new JSONObject();
try {
JSONAssert.assertEquals(gol.toString(), res.toString(), false);
o.put("result", "ok");
return o;
} catch (AssertionError E) {
o.put("differ", E.getMessage());
}
return o;
}
public static JSONArray run(Properties p, String... targets) throws Exception {
JSONArray summary = new JSONArray();
for (String target : targets) {
JSONObject result = new ServiceTest2(new File(target), p).test();
summary.put(result);
}
return summary;
}
static String getServicePath(String url) throws URISyntaxException {
// URIBuilder b = new URIBuilder("http://localhost:8080/csip-example/m/simpleservice/2.0");
URIBuilder b = new URIBuilder(url);
return b.getPath().substring(b.getPath().indexOf('/', 1));
}
public static void main(String[] args) throws Exception {
//// args = new String[]{"/od/projects/csip-all/csip-example/test/service_tests/all"};
// args = new String[]{"/od/tmp/csip-testing"};
// Properties config = new Properties();
// config.setProperty("concurrency", "1");
// config.setProperty("verify", "keyvalue");
//// config.setProperty("rounds", "3");
//// config.setProperty("delay", "3000");
// System.out.println(run(config, args).toString(4).replace("\\/", "/"));
}
}