ModelDataServiceCall.java [src/csip/api/client] Revision: default  Date:
/*
 * $Id$
 *
 * This file is part of the Cloud Services Integration Platform (CSIP),
 * a Model-as-a-Service framework, API and application suite.
 *
 * 2012-2022, Olaf David and others, OMSLab, Colorado State University.
 *
 * OMSLab licenses this file to you under the MIT license.
 * See the LICENSE file in the project root for more information.
 */
package csip.api.client;

import csip.utils.Client;
import csip.Config;
import csip.ModelDataService;
import csip.api.server.ServiceException;
import csip.utils.LRUCache;
import static csip.ModelDataService.KEY_VALUE;
import static csip.ModelDataService.KEY_NAME;
import csip.Utils;
import csip.utils.JSONUtils;
import csip.utils.Validation;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;

/**
 * ModelDataService Call. Preferred method to call a CSIP service from Java
 * using its fluent API. It represents a service request or response. The call
 * methods of a ModelDataServiceCall request returns a ModelDataServiceCall
 * response. This class implements the callable interface, supports client side
 * caching, performs call retries if the service transport layer fails, and
 * downloads result files if requested.
 * 
 * <p>
 * Example of an simple synchronous service call:
  <blockquote><pre>
   ModelDataServiceCall resp = new ModelDataServiceCall()
        .put("temp", 100, "C")  // add input data
        .withDefaultLogger()    // use the default logger
        .url("http://localhost:8080/csip-example/m/simpleservice/2.0")  // url
        .call();                // perform the call
    
    if (resp.serviceFinished()) {
      System.out.println(resp.get("temp"));
    }
   </pre></blockquote>
 * 
 * Example of an asynchronous service call:
 *  <blockquote><pre>
   ModelDataServiceCall resp = new ModelDataServiceCall()
        .put("temp", 100, "C")          // add input data
        .asAsync()                      // call async
        .withAsyncCallback(mds -@lt; {     // optional callback (e.g. to print progess)
          System.out.println(mds.getProgress());
        })
        .url("http://localhost:8080/csip-example/m/simpleservice/2.0")
        .call();
    
    if (resp.serviceFinished()) {
      System.out.println(resp.get("temp"));
    }
   </pre></blockquote>
 *
 * @author O David
 */
public final class ModelDataServiceCall implements Callable<ModelDataServiceCall> {

  ModelDataServiceCall parent;
  Map<String, Object> data;
  Map<String, String> header;
  JSONObject metainfo;
  List<File> att;
  String url;
  Logger log;
  int timeout = Client.DEF_TIMEOUT;
  int retries = 0;
  long retrysleep = 250;
  int firstPoll = -1;
  int nextPoll = -1;
  boolean callFailed;
  boolean sync = true;
  Consumer<ModelDataServiceCall> callback;

  boolean useCache;
  private static volatile LRUCache<String, JSONObject> cache;
  private static int cacheSize = 64;


  static LRUCache<String, JSONObject> cache() {
    LRUCache<String, JSONObject> ref = cache;
    if (ref == null) {
      synchronized (ModelDataServiceCall.class) {
        ref = cache;
        if (ref == null)
          cache = ref = new LRUCache<>(cacheSize);
      }
    }
    return ref;
  }

  public static final class ConfAccess {

    private ConfAccess() {
    }
  }
  private static final ConfAccess access = new ConfAccess();


  /**
   * Reset the static cache, will invalidate the current content.
   *
   * @param size
   */
  public static synchronized void setCacheSize(int size) {
    if (cacheSize == size)
      return;

    if (cache != null) {
      cache.clear();
      cache.setSize(size);
    }
    cacheSize = size;
  }


  public static ModelDataServiceCall fromJSON(File jsonRequest) throws Exception {
    return new ModelDataServiceCall(new JSONObject(FileUtils.readFileToString(jsonRequest, "UTF-8")), null);
  }


  public static ModelDataServiceCall fromJSON(URL jsonRequest) throws Exception {
    return new ModelDataServiceCall(new JSONObject(IOUtils.toString(jsonRequest, "UTF-8")), null);
  }


  public static ModelDataServiceCall fromJSON(String jsonRequest) throws Exception {
    return new ModelDataServiceCall(new JSONObject(jsonRequest), null);
  }


  ModelDataServiceCall(Map<String, Object> data, JSONObject metainfo, boolean callFailed) {
    this.data = data;
    this.metainfo = metainfo;
    this.callFailed = callFailed;
  }


  ModelDataServiceCall(JSONObject rr, ModelDataServiceCall parent) throws JSONException {
    this(fromJSON(
        rr.has("result") ? rr.getJSONArray("result")
        : (rr.has("parameter") ? rr.getJSONArray("parameter")
        : new JSONArray())),
        new JSONObject((rr.has("metainfo") ? rr.getJSONObject("metainfo") : rr).toString()),
        !rr.has("metainfo")
    );
    if (parent != null) {
      this.parent = parent;
      log = parent.log;
      url = parent.url;
      useCache = parent.useCache;
      timeout = parent.timeout;
      retries = parent.retries;
      retrysleep = parent.retrysleep;
    }
  }


  /**
   * Create a new Call.
   */
  public ModelDataServiceCall() {
    this(new LinkedHashMap<>(), new JSONObject(), false);
  }


  public ModelDataServiceCall put(String name, Collection value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, String type, JSONArray coordinates) {
    try {
      JSONObject geom = new JSONObject();
      geom.put(KEY_NAME, name);
      geom.put("type", type);
      geom.put("coordinates", coordinates);
      data.put(name, geom);
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, JSONObject value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, JSONArray value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, String value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, String[] value) {
    try {
      data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, boolean value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, boolean[] value) {
    try {
      data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, int value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, int[] value) {
    try {
      data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, double value) {
    try {
      data.put(name, JSONUtils.data(name, value));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, double value, String unit) {
    try {
      data.put(name, JSONUtils.data(name, value, unit));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, double[] value) {
    try {
      data.put(name, JSONUtils.data(name, JSONUtils.toArray(value)));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  public ModelDataServiceCall put(String name, double[] value, String unit) {
    try {
      data.put(name, JSONUtils.data(name, JSONUtils.toArray(value), unit));
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  /**
   * Attach files to the request.
   *
   * @param files the list of files for attach.
   * @return this ModelDataServiceCall instance
   */
  public synchronized ModelDataServiceCall attach(File... files) {
    if (att == null)
      att = new ArrayList<>();

    for (File f : files) {
      if (!f.exists() && !f.canRead())
        throw new IllegalArgumentException("File not found or cannot read: " + f);

      att.add(f);
    }
    return this;
  }


  /**
   * Add a meta data entry.
   *
   * @param key the key
   * @param value the value
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall putMeta(String key, String value) {
    try {
      metainfo.put(key, value);
    } catch (JSONException ex) {
      throw new IllegalArgumentException(ex);
    }
    return this;
  }


  /**
   * The url to call.
   *
   * @param url the url
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall url(String url) {
    if (url == null || url.isEmpty())
      throw new IllegalArgumentException("No url provided.");

    try {
      url = url.trim();
      new URL(url).toURI();
    } catch (MalformedURLException | URISyntaxException ex) {
      throw new IllegalArgumentException("Illegal Url: " + url);
    }
    this.url = url;
    return this;
  }


  /**
   * Use the default system logger.
   *
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withDefaultLogger() {
    return withLogger(Config.getSystemLogger(access));
  }


  /**
   * Use custom polling, overwrite the values provided by the service. Use with
   * care!
   *
   * @param first the first poll in ms
   * @param next any subsequent poll in ms
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withPolling(int first, int next) {
    firstPoll = first;
    nextPoll = next;
    return this;
  }


  /**
   * Use a custom logger.
   *
   * @param l the logger
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withLogger(Logger l) {
    if (l == null)
      throw new NullPointerException("logger");

    this.log = l;
    return this;
  }


  /**
   * Cache the service calls.
   *
   * @param cache true if calls should be cached.
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withCache(boolean cache) {
    this.useCache = cache;
    return this;
  }


  /**
   * Call the service asynchronously. (Default is synchronously)
   *
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall asAsync() {
    this.sync = false;
    return this;
  }


  /**
   * Call the service asynchronously or synchronously. (Default is
   * synchronously)
   *
   * @param sync true if synchronously, false otherwise
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall asSync(boolean sync) {
    this.sync = sync;
    return this;
  }


  /**
   * Set a callback consumer. It gets called on every async poll.
   *
   * @param callback the consumer
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withAsyncCallback(
      Consumer<ModelDataServiceCall> callback) {
    this.callback = callback;
    return this;
  }


  /**
   * Set the timeout.
   *
   * @param timeout the timeout.
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withTimeout(int timeout) {
    if (timeout < 0)
      throw new IllegalArgumentException("timeout >= 0, was: " + timeout);

    this.timeout = timeout;
    return this;
  }


  /**
   * Add a custom header.
   *
   * @param key the header key
   * @param value header value
   * @return this ModelDataServiceCall instance
   */
  public synchronized ModelDataServiceCall withHeader(String key, String value) {
    if (header == null)
      header = new LinkedHashMap<>();

    header.put(key, value);
    return this;
  }


  /**
   * Number of retries for a call, (default is 3)
   *
   * @param retries number of retries
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withRetries(int retries) {
    if (retries < 0)
      throw new IllegalArgumentException("retries >= 0!, was " + retries);

    this.retries = retries;
    return this;
  }


  /**
   * Pause in ms between retries. (default is 250)
   *
   * @param sleep the pause in ms.
   * @return this ModelDataServiceCall instance
   */
  public ModelDataServiceCall withRetryPause(long sleep) {
    if (sleep < 0)
      throw new IllegalArgumentException("retrypause >= 0, was " + sleep);

    retrysleep = sleep;
    return this;
  }


  /**
   * get the request as JSON
   *
   * @return the assembled JSON request.
   */
  public JSONObject request() {
    JSONArray param = asJSON();
    JSONObject req = JSONUtils.newRequest(param, metainfo);
    return req;
  }


  /**
   * Call the service.
   *
   * @return the response as ModelDataServiceCall
   * @throws Exception if something goes wrong
   */
  @Override
  public ModelDataServiceCall call() throws Exception {
    if (url == null)
      throw new RuntimeException("No url provided.");

    JSONArray param = asJSON();
    if (!sync)
      metainfo.put("mode", "async");

    JSONObject req = JSONUtils.newRequest(param, metainfo);
    JSONObject res;
    String hash = null;

    if (useCache) {
//      in = in.replace(" ", "").replace("\n", "").replace("\t", "");
      hash = Validation.digest(url + "-" + param.toString(), "MD5");
      if (hash != null) {
        res = cache().get(hash);
        if (res != null) {
          if (log != null && log.isLoggable(Level.INFO))
            log.info("Fetched response from cache: " + res);

          return new ModelDataServiceCall(res, this);
        }
      }
    }

    File[] files = (att == null) ? null : att.toArray(new File[0]);
    int ret = retries;
    ModelDataServiceCall result;

    try (Client cl = new Client(timeout, log)) {
      do {
        if (ret != retries)  // skip this in the first round.
          Thread.sleep(retrysleep);

        res = cl.doPOST(url, req, files, header);
        result = new ModelDataServiceCall(res, this);
      } while (ret-- > 0 && result.callFailed());
    }

    if (!sync) {
      if (result.serviceSubmitted()) {
        int fp = firstPoll > 0 ? firstPoll : result.getFirstPoll();
        int np = nextPoll > 0 ? nextPoll : result.getNextPoll();

        Thread.sleep(fp);
        try (Client cl = new Client(timeout, log)) {
          String[] u = Utils.getURIParts(result.url);
          String suid = result.getSUID();
          if (suid == null)
            throw new ServiceException("No valid SUID for async call: " + url);

          String qUrl = u[0] + u[1] + u[2] + "/" + u[3] + "/q/" + suid;
          boolean running = false;
          do {
            if (running) // skip it the first time
              Thread.sleep(np);

            if (log != null && log.isLoggable(Level.INFO))
              log.info("GET: " + qUrl);

            String resp = cl.doGET(qUrl);
            res = new JSONObject(resp);
            result = new ModelDataServiceCall(res, this);
            if (callback != null)
              callback.accept(result);

            running = result.serviceRunning();
          } while (running);
        }
      }
    }

    if (useCache && !result.callFailed()) {
      if (log != null && log.isLoggable(Level.INFO))
        log.info("Adding to cache:" + hash + " for:  " + url + "-" + req.toString());

      cache().put(hash, res);
    }
    return result;
  }


//// response methods.  
  public List<String> getNames() {
    return new ArrayList<>(data.keySet());
  }

  public int getCount() {
    return data.size();
  }


  public void download(String name, File file) throws Exception {
    String url = getString(name);
    try {
      new URL(url).toURI();
    } catch (MalformedURLException | URISyntaxException E) {
      throw new IllegalArgumentException("Malformed Url: " + url);
    } 
    try (Client c = new Client(timeout, log)) {
      c.doGET(url, file);
    }
  }


  public String download(String name) throws Exception {
    String url = getString(name);
    try {
      new URL(url).toURI();
    } catch (MalformedURLException | URISyntaxException E) {
      throw new IllegalArgumentException("Malformed Url: " + url);
    }
    String content;
    try (Client c = new Client(timeout, log)) {
      content = c.doGET(url);
    }
    return content;
  }


  public Object get(String name) {
    try {
      return getJSONObjectInMap(name).get(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public Object get(String name, Object def) {
    try {
      return getJSONObjectInMap(name).get(KEY_VALUE);
    } catch (JSONException | IllegalArgumentException ex) {
      return def;
    }
  }


  public double getDouble(String name) {
    try {
      return getJSONObjectInMap(name).getDouble(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public double getDouble(String name, double def) {
    return getJSONObjectInMap(name).optDouble(KEY_VALUE, def);
  }


  public int getInt(String name) {
    try {
      return getJSONObjectInMap(name).getInt(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public int getInt(String name, int def) {
    return getJSONObjectInMap(name).optInt(KEY_VALUE, def);
  }


  public long getLong(String name) {
    try {
      return getJSONObjectInMap(name).getLong(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public long getLong(String name, long def) {
    return getJSONObjectInMap(name).optLong(KEY_VALUE, def);
  }


  public boolean getBoolean(String name) {
    try {
      return getJSONObjectInMap(name).getBoolean(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public boolean getBoolean(String name, boolean def) {
    return getJSONObjectInMap(name).optBoolean(KEY_VALUE, def);
  }


  public String getString(String name) {
    try {
      return getJSONObjectInMap(name).getString(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public String getString(String name, String def) {
    return getJSONObjectInMap(name).optString(KEY_VALUE, def);
  }


  public JSONObject getJSONObject(String name) {
    try {
      return getJSONObjectInMap(name).getJSONObject(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public JSONObject getJSONObject(String name, JSONObject def) {
    try {
      return getJSONObjectInMap(name).getJSONObject(KEY_VALUE);
    } catch (JSONException | IllegalArgumentException ex) {
      return def;
    }
  }


  public JSONArray getJSONArray(String name) {
    try {
      return getJSONObjectInMap(name).getJSONArray(KEY_VALUE);
    } catch (JSONException ex) {
      throw new IllegalArgumentException("'" + name + "': " + ex.getMessage());
    }
  }


  public JSONArray getJSONArray(String name, JSONArray def) {
    try {
      return getJSONObjectInMap(name).getJSONArray(KEY_VALUE);
    } catch (JSONException | IllegalArgumentException ex) {
      return def;
    }
  }


  public boolean has(String name) {
    return data.containsKey(name);
  }


  // Metainfo
  public boolean hasMeta(String name) {
    return metainfo.has(name);
  }


  public Object getMeta(String name) {
    return metainfo.opt(name);
  }


  public String getMetaString(String key) {
    return metainfo.optString(key);
  }


  /**
   * Get the metainfo 'error' value.
   *
   * @return the error string, or 'null' if there is none.
   */
  public String getError() {
    return metainfo.optString(ModelDataService.ERROR, null);
  }


  /**
   * Get the SUID.
   *
   * @return the suid, '0' if there is none.
   */
  public String getSUID() {
    return metainfo.optString(ModelDataService.KEY_SUUID, null);
  }


  /**
   * Get the frist poll value.
   *
   * @return the time in ms. (2000 if there is none)
   */
  public int getFirstPoll() {
    return metainfo.optInt(ModelDataService.KEY_FIRST_POLL, 2000);
  }


  /**
   * Get the next poll value.
   *
   * @return the time in ms. (2000 if there is none)
   */
  public int getNextPoll() {
    return metainfo.optInt(ModelDataService.KEY_NEXT_POLL, 2000);
  }


  /**
   * Get the progress string (for async calls.)
   *
   * @return the progress, or "" if there is none.
   */
  public String getProgress() {
    return metainfo.optString(ModelDataService.KEY_PROGRESS, "");
  }


  /**
   * check this it service transport failed (http).
   *
   * @return
   */
  public boolean callFailed() {
    return callFailed;
  }


  public boolean serviceFinished() {
    return !callFailed && isStatus(ModelDataService.FINISHED);
  }


  public boolean serviceSubmitted() {
    return !callFailed && isStatus(ModelDataService.SUBMITTED);
  }


  public boolean serviceRunning() {
    return !callFailed && isStatus(ModelDataService.RUNNING);
  }


  public boolean serviceFailed() {
    return !callFailed && isStatus(ModelDataService.FAILED);
  }


  public boolean serviceCancelled() {
    return !callFailed && isStatus(ModelDataService.CANCELED);
  }


  public boolean serviceTimeout() {
    return !callFailed && isStatus(ModelDataService.TIMEDOUT);
  }


  public boolean serviceReturned() {
    return !callFailed && metainfo.has(ModelDataService.KEY_STATUS);
  }


  private boolean isStatus(String st) {
    return st.equals(metainfo.optString(ModelDataService.KEY_STATUS));
  }


  public ModelDataServiceCall getParent() {
    return parent;
  }


  @Override
  public String toString() {
    try {
      String a = "Metainfo: \n" + metainfo.toString(2);
      return a + "\nData:\n" + asJSON().toString(2);
    } catch (JSONException E) {
      return asJSON().toString();
    }
  }


  ///////////////////// private helper methods
  private JSONObject getJSONObjectInMap(String name) {
    Object o = data.get(name);
    if (o != null)
      return (JSONObject) o;

    throw new IllegalArgumentException("Data not not found for: " + name);
  }


  static Map<String, Object> fromJSON(JSONArray data) throws JSONException {
    Map<String, Object> p = new LinkedHashMap<>();
    for (int i = 0; i < data.length(); i++) {
      JSONObject o = data.getJSONObject(i);
      String name = o.getString(KEY_NAME);
      p.put(name, o);
    }
    return p;
  }


  JSONArray asJSON() {
    JSONArray a = new JSONArray();
    for (Object o : data.values()) {
      if (o instanceof JSONObject) {
        a.put((JSONObject) o);
      } else if (o instanceof ModelDataServiceCall) {
        ModelDataServiceCall o_ = (ModelDataServiceCall) o;
        JSONObject param = new JSONObject();
        try {
          param.put(KEY_NAME, "m");
          param.put(KEY_VALUE, o_.asJSON());
        } catch (JSONException ex) {
          throw new IllegalArgumentException(ex);
        }
        a.put(param);
      }
    }
    return a;
  }
}