ModelDataServiceCall.java [src/csip/api/client] Revision: Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.api.client;
import csip.utils.Client;
import csip.Config;
import csip.ModelDataService;
import csip.api.server.ServiceException;
import csip.utils.LRUCache;
import static csip.ModelDataService.KEY_VALUE;
import static csip.ModelDataService.KEY_NAME;
import csip.Utils;
import csip.utils.JSONUtils;
import csip.utils.Validation;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
/**
* ModelDataService Call. Preferred method to call a CSIP service from Java
* using its fluent API. It represents a service request or response. The call
* methods of a ModelDataServiceCall request returns a ModelDataServiceCall
* response. This class implements the callable interface, supports client side
* caching, performs call retries if the service transport layer fails, and
* downloads result files if requested.
*
* <p>
* Example of an simple synchronous service call:
<blockquote><pre>
ModelDataServiceCall resp = new ModelDataServiceCall()
.put("temp", 100, "C") // add input data
.withDefaultLogger() // use the default logger
.url("http://localhost:8080/csip-example/m/simpleservice/2.0") // url
.call(); // perform the call
if (resp.serviceFinished()) {
System.out.println(resp.get("temp"));
}
</pre></blockquote>
*
* Example of an asynchronous service call:
* <blockquote><pre>
ModelDataServiceCall resp = new ModelDataServiceCall()
.put("temp", 100, "C") // add input data
.asAsync() // call async
.withAsyncCallback(mds -@lt; { // optional callback (e.g. to print progess)
System.out.println(mds.getProgress());
})
.url("http://localhost:8080/csip-example/m/simpleservice/2.0")
.call();
if (resp.serviceFinished()) {
System.out.println(resp.get("temp"));
}
</pre></blockquote>
*
* @author O David
*/
public final class ModelDataServiceCall implements Callable<ModelDataServiceCall> {
ModelDataServiceCall parent;
Map<String, Object> data;
Map<String, String> header;
JSONObject metainfo;
List<File> att;
String url;
Logger log;
int timeout = Client.DEF_TIMEOUT;
int retries = 0;
long retrysleep = 250;
int firstPoll = -1;
int nextPoll = -1;
boolean callFailed;
boolean sync = true;
Consumer<ModelDataServiceCall> callback;
boolean useCache;
private static volatile LRUCache<String, JSONObject> cache;
private static int cacheSize = 64;
static LRUCache<String, JSONObject> cache() {
LRUCache<String, JSONObject> ref = cache;
if (ref == null) {
synchronized (ModelDataServiceCall.class) {
ref = cache;
if (ref == null)
cache = ref = new LRUCache<>(cacheSize);
}
}
return ref;
}
public static final class ConfAccess {
private ConfAccess() {
}
}
private static final ConfAccess access = new ConfAccess();
/**
* Reset the static cache, will invalidate the current content.
*
* @param size
*/
public static synchronized void setCacheSize(int size) {
if (cacheSize == size)
return;
if (cache != null) {
cache.clear();
cache.setSize(size);
}
cacheSize = size;
}
public static ModelDataServiceCall fromJSON(File jsonRequest) throws Exception {
return new ModelDataServiceCall(new JSONObject(FileUtils.readFileToString(jsonRequest, "UTF-8")), null);
}
public static ModelDataServiceCall fromJSON(URL jsonRequest) throws Exception {
return new ModelDataServiceCall(new JSONObject(IOUtils.toString(jsonRequest, "UTF-8")), null);
}
public static ModelDataServiceCall fromJSON(String jsonRequest) throws Exception {
return new ModelDataServiceCall(new JSONObject(jsonRequest), null);
}
ModelDataServiceCall(Map<String, Object> data, JSONObject metainfo, boolean callFailed) {
this.data = data;
this.metainfo = metainfo;
this.callFailed = callFailed;
}
ModelDataServiceCall(JSONObject rr, ModelDataServiceCall parent) throws JSONException {
this(fromJSON(
rr.has("result") ? rr.getJSONArray("result")
: (rr.has("parameter") ? rr.getJSONArray("parameter")
: new JSONArray())),
new JSONObject((rr.has("metainfo") ? rr.getJSONObject("metainfo") : rr).toString()),
!rr.has("metainfo")
);
if (parent != null) {
this.parent = parent;
log = parent.log;
url = parent.url;
useCache = parent.useCache;
timeout = parent.timeout;
retries = parent.retries;
retrysleep = parent.retrysleep;
}
}
/**
* Create a new Call.
*/
public ModelDataServiceCall() {
this(new LinkedHashMap<>(), new JSONObject(), false);
}
public ModelDataServiceCall put(String name, Collection value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, String type, JSONArray coordinates) {
try {
JSONObject geom = new JSONObject();
geom.put(KEY_NAME, name);
geom.put("type", type);
geom.put("coordinates", coordinates);
data.put(name, geom);
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, JSONObject value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, JSONArray value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, String value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, String[] value) {
try {
data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, boolean value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, boolean[] value) {
try {
data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, int value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, int[] value) {
try {
data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, double value) {
try {
data.put(name, JSONUtils.data(name, value));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, double value, String unit) {
try {
data.put(name, JSONUtils.data(name, value, unit));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, double[] value) {
try {
data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
public ModelDataServiceCall put(String name, double[] value, String unit) {
try {
data.put(name, JSONUtils.data(name, JSONUtils.toArray(value), unit));
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
/**
* Attach files to the request.
*
* @param files the list of files for attach.
* @return this ModelDataServiceCall instance
*/
public synchronized ModelDataServiceCall attach(File... files) {
if (att == null)
att = new ArrayList<>();
for (File f : files) {
if (!f.exists() && !f.canRead())
throw new IllegalArgumentException("File not found or cannot read: " + f);
att.add(f);
}
return this;
}
/**
* Add a meta data entry.
*
* @param key the key
* @param value the value
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall putMeta(String key, String value) {
try {
metainfo.put(key, value);
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
return this;
}
/**
* The url to call.
*
* @param url the url
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall url(String url) {
if (url == null || url.isEmpty())
throw new IllegalArgumentException("No url provided.");
try {
url = url.trim();
new URL(url).toURI();
} catch (MalformedURLException | URISyntaxException ex) {
throw new IllegalArgumentException("Illegal Url: " + url);
}
this.url = url;
return this;
}
/**
* Use the default system logger.
*
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withDefaultLogger() {
return withLogger(Config.getSystemLogger(access));
}
/**
* Use custom polling, overwrite the values provided by the service. Use with
* care!
*
* @param first the first poll in ms
* @param next any subsequent poll in ms
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withPolling(int first, int next) {
firstPoll = first;
nextPoll = next;
return this;
}
/**
* Use a custom logger.
*
* @param l the logger
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withLogger(Logger l) {
if (l == null)
throw new NullPointerException("logger");
this.log = l;
return this;
}
/**
* Cache the service calls.
*
* @param cache true if calls should be cached.
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withCache(boolean cache) {
this.useCache = cache;
return this;
}
/**
* Call the service asynchronously. (Default is synchronously)
*
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall asAsync() {
this.sync = false;
return this;
}
/**
* Call the service asynchronously or synchronously. (Default is
* synchronously)
*
* @param sync true if synchronously, false otherwise
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall asSync(boolean sync) {
this.sync = sync;
return this;
}
/**
* Set a callback consumer. It gets called on every async poll.
*
* @param callback the consumer
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withAsyncCallback(
Consumer<ModelDataServiceCall> callback) {
this.callback = callback;
return this;
}
/**
* Set the timeout.
*
* @param timeout the timeout.
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withTimeout(int timeout) {
if (timeout < 0)
throw new IllegalArgumentException("timeout >= 0, was: " + timeout);
this.timeout = timeout;
return this;
}
/**
* Add a custom header.
*
* @param key the header key
* @param value header value
* @return this ModelDataServiceCall instance
*/
public synchronized ModelDataServiceCall withHeader(String key, String value) {
if (header == null)
header = new LinkedHashMap<>();
header.put(key, value);
return this;
}
/**
* Number of retries for a call, (default is 3)
*
* @param retries number of retries
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withRetries(int retries) {
if (retries < 0)
throw new IllegalArgumentException("retries >= 0!, was " + retries);
this.retries = retries;
return this;
}
/**
* Pause in ms between retries. (default is 250)
*
* @param sleep the pause in ms.
* @return this ModelDataServiceCall instance
*/
public ModelDataServiceCall withRetryPause(long sleep) {
if (sleep < 0)
throw new IllegalArgumentException("retrypause >= 0, was " + sleep);
retrysleep = sleep;
return this;
}
/**
* get the request as JSON
*
* @return the assembled JSON request.
*/
public JSONObject request() {
JSONArray param = asJSON();
JSONObject req = JSONUtils.newRequest(param, metainfo);
return req;
}
/**
* Call the service.
*
* @return the response as ModelDataServiceCall
* @throws Exception if something goes wrong
*/
@Override
public ModelDataServiceCall call() throws Exception {
if (url == null)
throw new RuntimeException("No url provided.");
JSONArray param = asJSON();
if (!sync)
metainfo.put("mode", "async");
JSONObject req = JSONUtils.newRequest(param, metainfo);
JSONObject res;
String hash = null;
if (useCache) {
// in = in.replace(" ", "").replace("\n", "").replace("\t", "");
hash = Validation.digest(url + "-" + param.toString(), "MD5");
if (hash != null) {
res = cache().get(hash);
if (res != null) {
if (log != null && log.isLoggable(Level.INFO))
log.info("Fetched response from cache: " + res);
return new ModelDataServiceCall(res, this);
}
}
}
File[] files = (att == null) ? null : att.toArray(new File[0]);
int ret = retries;
ModelDataServiceCall result;
try (Client cl = new Client(timeout, log)) {
do {
if (ret != retries) // skip this in the first round.
Thread.sleep(retrysleep);
res = cl.doPOST(url, req, files, header);
result = new ModelDataServiceCall(res, this);
} while (ret-- > 0 && result.callFailed());
}
if (!sync) {
if (result.serviceSubmitted()) {
int fp = firstPoll > 0 ? firstPoll : result.getFirstPoll();
int np = nextPoll > 0 ? nextPoll : result.getNextPoll();
Thread.sleep(fp);
try (Client cl = new Client(timeout, log)) {
String[] u = Utils.getURIParts(result.url);
String suid = result.getSUID();
if (suid == null)
throw new ServiceException("No valid SUID for async call: " + url);
String qUrl = u[0] + u[1] + u[2] + "/" + u[3] + "/q/" + suid;
boolean running = false;
do {
if (running) // skip it the first time
Thread.sleep(np);
if (log != null && log.isLoggable(Level.INFO))
log.info("GET: " + qUrl);
String resp = cl.doGET(qUrl);
res = new JSONObject(resp);
result = new ModelDataServiceCall(res, this);
if (callback != null)
callback.accept(result);
running = result.serviceRunning();
} while (running);
}
}
}
if (useCache && !result.callFailed()) {
if (log != null && log.isLoggable(Level.INFO))
log.info("Adding to cache:" + hash + " for: " + url + "-" + req.toString());
cache().put(hash, res);
}
return result;
}
//// response methods.
public List<String> getNames() {
return new ArrayList<>(data.keySet());
}
public int getCount() {
return data.size();
}
public void download(String name, File file) throws Exception {
String url = getString(name);
try {
new URL(url).toURI();
} catch (MalformedURLException | URISyntaxException E) {
throw new IllegalArgumentException("Malformed Url: " + url);
}
try (Client c = new Client(timeout, log)) {
c.doGET(url, file);
}
}
public String download(String name) throws Exception {
String url = getString(name);
try {
new URL(url).toURI();
} catch (MalformedURLException | URISyntaxException E) {
throw new IllegalArgumentException("Malformed Url: " + url);
}
String content;
try (Client c = new Client(timeout, log)) {
content = c.doGET(url);
}
return content;
}
public Object get(String name) {
try {
return getJSONObjectInMap(name).get(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public Object get(String name, Object def) {
try {
return getJSONObjectInMap(name).get(KEY_VALUE);
} catch (JSONException | IllegalArgumentException ex) {
return def;
}
}
public double getDouble(String name) {
try {
return getJSONObjectInMap(name).getDouble(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public double getDouble(String name, double def) {
return getJSONObjectInMap(name).optDouble(KEY_VALUE, def);
}
public int getInt(String name) {
try {
return getJSONObjectInMap(name).getInt(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public int getInt(String name, int def) {
return getJSONObjectInMap(name).optInt(KEY_VALUE, def);
}
public long getLong(String name) {
try {
return getJSONObjectInMap(name).getLong(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public long getLong(String name, long def) {
return getJSONObjectInMap(name).optLong(KEY_VALUE, def);
}
public boolean getBoolean(String name) {
try {
return getJSONObjectInMap(name).getBoolean(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public boolean getBoolean(String name, boolean def) {
return getJSONObjectInMap(name).optBoolean(KEY_VALUE, def);
}
public String getString(String name) {
try {
return getJSONObjectInMap(name).getString(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public String getString(String name, String def) {
return getJSONObjectInMap(name).optString(KEY_VALUE, def);
}
public JSONObject getJSONObject(String name) {
try {
return getJSONObjectInMap(name).getJSONObject(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public JSONObject getJSONObject(String name, JSONObject def) {
try {
return getJSONObjectInMap(name).getJSONObject(KEY_VALUE);
} catch (JSONException | IllegalArgumentException ex) {
return def;
}
}
public JSONArray getJSONArray(String name) {
try {
return getJSONObjectInMap(name).getJSONArray(KEY_VALUE);
} catch (JSONException ex) {
throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
}
}
public JSONArray getJSONArray(String name, JSONArray def) {
try {
return getJSONObjectInMap(name).getJSONArray(KEY_VALUE);
} catch (JSONException | IllegalArgumentException ex) {
return def;
}
}
public boolean has(String name) {
return data.containsKey(name);
}
// Metainfo
public boolean hasMeta(String name) {
return metainfo.has(name);
}
public Object getMeta(String name) {
return metainfo.opt(name);
}
public String getMetaString(String key) {
return metainfo.optString(key);
}
/**
* Get the metainfo 'error' value.
*
* @return the error string, or 'null' if there is none.
*/
public String getError() {
return metainfo.optString(ModelDataService.ERROR, null);
}
/**
* Get the SUID.
*
* @return the suid, '0' if there is none.
*/
public String getSUID() {
return metainfo.optString(ModelDataService.KEY_SUUID, null);
}
/**
* Get the frist poll value.
*
* @return the time in ms. (2000 if there is none)
*/
public int getFirstPoll() {
return metainfo.optInt(ModelDataService.KEY_FIRST_POLL, 2000);
}
/**
* Get the next poll value.
*
* @return the time in ms. (2000 if there is none)
*/
public int getNextPoll() {
return metainfo.optInt(ModelDataService.KEY_NEXT_POLL, 2000);
}
/**
* Get the progress string (for async calls.)
*
* @return the progress, or "" if there is none.
*/
public String getProgress() {
return metainfo.optString(ModelDataService.KEY_PROGRESS, "");
}
/**
* check this it service transport failed (http).
*
* @return
*/
public boolean callFailed() {
return callFailed;
}
public boolean serviceFinished() {
return !callFailed && isStatus(ModelDataService.FINISHED);
}
public boolean serviceSubmitted() {
return !callFailed && isStatus(ModelDataService.SUBMITTED);
}
public boolean serviceRunning() {
return !callFailed && isStatus(ModelDataService.RUNNING);
}
public boolean serviceFailed() {
return !callFailed && isStatus(ModelDataService.FAILED);
}
public boolean serviceCancelled() {
return !callFailed && isStatus(ModelDataService.CANCELED);
}
public boolean serviceTimeout() {
return !callFailed && isStatus(ModelDataService.TIMEDOUT);
}
public boolean serviceReturned() {
return !callFailed && metainfo.has(ModelDataService.KEY_STATUS);
}
private boolean isStatus(String st) {
return st.equals(metainfo.optString(ModelDataService.KEY_STATUS));
}
public ModelDataServiceCall getParent() {
return parent;
}
@Override
public String toString() {
try {
String a = "Metainfo: \n" + metainfo.toString(2);
return a + "\nData:\n" + asJSON().toString(2);
} catch (JSONException E) {
return asJSON().toString();
}
}
///////////////////// private helper methods
private JSONObject getJSONObjectInMap(String name) {
Object o = data.get(name);
if (o != null)
return (JSONObject) o;
throw new IllegalArgumentException("Data not not found for: " + name);
}
static Map<String, Object> fromJSON(JSONArray data) throws JSONException {
Map<String, Object> p = new LinkedHashMap<>();
for (int i = 0; i < data.length(); i++) {
JSONObject o = data.getJSONObject(i);
String name = o.getString(KEY_NAME);
p.put(name, o);
}
return p;
}
JSONArray asJSON() {
JSONArray a = new JSONArray();
for (Object o : data.values()) {
if (o instanceof JSONObject) {
a.put((JSONObject) o);
} else if (o instanceof ModelDataServiceCall) {
ModelDataServiceCall o_ = (ModelDataServiceCall) o;
JSONObject param = new JSONObject();
try {
param.put(KEY_NAME, "m");
param.put(KEY_VALUE, o_.asJSON());
} catch (JSONException ex) {
throw new IllegalArgumentException(ex);
}
a.put(param);
}
}
return a;
}
}