ExternalService.java [src/java/svap/utils] Revision: default Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2017, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package svap.utils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import static java.net.HttpURLConnection.HTTP_OK;
import java.net.URL;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.pool.PoolStats;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
/**
* HTTP Client for easier external REST service calling. Use this object instead
* of csip.Client when you are calling external RESTFul services that do not
* expect multiplart form requests with the JSON labeled as "param" in the
* request payload.
*
* @author od
* @author <a href="mailto:shaun.case@colostate.edu">Shaun Case</a>*
*/
public class ExternalService {
private Logger log;
private static final PoolingHttpClientConnectionManager cm;
private CloseableHttpClient httpClient;
static final int DEF_TIMEOUT = 360; // 5 minutes timeout
private CredentialsProvider creds;
private BasicHttpContext context;
static {
cm = new PoolingHttpClientConnectionManager(
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", new SSLConnectionSocketFactory(SSLContexts.createSystemDefault()))
.build()
);
cm.setMaxTotal(250);
// Increase default max connection per route to 75
cm.setDefaultMaxPerRoute(100);
// Increase max connections for localhost:80 to 125
//HttpHost localhost = new HttpHost("locahost", 80);
//cm.setMaxPerRoute(new HttpRoute(localhost), 150);
}
/**
* Create a Client
*/
public ExternalService() {
this(DEF_TIMEOUT, Logger.getLogger(ExternalService.class.getName()));
}
/**
* Create a client with a logger.
*
* @param log the Logger
*
*/
public ExternalService(Logger log) {
this(DEF_TIMEOUT, log);
}
/**
* Create a client with a custom timeout.
*
* @param timeout the timeout in
*/
public ExternalService(int timeout) {
this(timeout, Logger.getLogger(ExternalService.class.getName()));
}
public ExternalService(int timeout, Logger log) {
RequestConfig reqConfig = init(timeout, log);
httpClient = HttpClientBuilder.create()
.setDefaultRequestConfig(reqConfig)
.setConnectionManager(cm)
.build();
}
public ExternalService(int timeout, Logger log, String username, String password) {
RequestConfig reqConfig = init(timeout, log);
context = new BasicHttpContext();
BasicScheme basicAuth = new BasicScheme();
context.setAttribute("preemptive-auth", basicAuth);
creds = new BasicCredentialsProvider();
creds.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
httpClient = HttpClientBuilder.create()
.setDefaultRequestConfig(reqConfig)
.setConnectionManager(cm)
.addInterceptorFirst(new PreemptiveAuthInterceptor())
.build();
}
private RequestConfig init(int timeout, Logger log) {
this.log = log;
RequestConfig reqConfig = RequestConfig.custom()
.setConnectTimeout(timeout * 1000)
.setConnectionRequestTimeout(timeout * 1000)
.setSocketTimeout(timeout * 1000) // 10 minute max per model run
.setCookieSpec(CookieSpecs.BEST_MATCH)
.setExpectContinueEnabled(true)
.setStaleConnectionCheckEnabled(true)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.build();
return reqConfig;
}
private class PreemptiveAuthInterceptor implements HttpRequestInterceptor {
@Override
@SuppressWarnings("deprecation")
public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
request.addHeader(BasicScheme.authenticate(creds.getCredentials(AuthScope.ANY), "US-ASCII", false));
}
}
/**
* Pings a HTTP URL. This effectively sends a HEAD request and returns
* <code>true</code> if the response code is in the 200-399 range.
*
* @param url The HTTP URL to be pinged.
* @param timeout The timeout in millis for both the connection timeout and
* the response read timeout. Note that the total timeout is effectively two
* times the given timeout.
* @return the time to respond if the given HTTP URL has returned response
* code 200-399 on a HEAD request within the given timeout, otherwise
* <code>-1</code>.
*/
public static long ping(String url, int timeout) {
url = url.replaceFirst("^https", "http");
try {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
connection.setConnectTimeout(timeout);
connection.setReadTimeout(timeout);
connection.setRequestMethod("HEAD");
long start = System.currentTimeMillis();
int responseCode = connection.getResponseCode();
long end = System.currentTimeMillis();
return (200 <= responseCode && responseCode <= 399) ? (end - start) : -1;
} catch (Exception E) {
return -1;
}
}
public File doGET(String url, File file) throws Exception {
if (log != null && log.isLoggable(Level.INFO)) {
log.info("GET : " + url + " -> " + file.toString());
}
HttpGet get = new HttpGet(url);
try (CloseableHttpResponse resultsResponse = httpClient.execute(get)) {
InputStream output = resultsResponse.getEntity().getContent();
FileUtils.copyInputStreamToFile(output, file);
} finally {
get.releaseConnection();
}
return file;
}
public String doGET(String url) throws Exception {
if (log != null && log.isLoggable(Level.INFO)) {
log.info("GET : " + url);
}
HttpGet get = new HttpGet(url);
try (CloseableHttpResponse resultsResponse = httpClient.execute(get)) {
InputStream output = resultsResponse.getEntity().getContent();
return IOUtils.toString(output);
} finally {
get.releaseConnection();
}
}
public int doDelete(String url) throws Exception {
HttpDelete del = new HttpDelete(url);
int status = 0;
try {
HttpResponse resultsResponse = httpClient.execute(del);
status = resultsResponse.getStatusLine().getStatusCode();
} finally {
del.releaseConnection();
}
if (log != null && log.isLoggable(Level.INFO)) {
log.info("DELETE : " + url + " -> " + status);
}
return status;
}
/**
* HTTP Post with the request JSON. Does not assume any specific JSON
* structure to be sent. The JSON request should be completed by the calling
* code.
*
* @param uri the endpoint
* @param req the request JSON
* @return the response JSON
* @throws Exception if something goes wrong
*/
public JSONObject doPOST(String uri, JSONObject req) throws Exception {
EntityBuilder builder = EntityBuilder.create();
builder.setText(req.toString());
builder.setContentType(ContentType.APPLICATION_JSON);
HttpPost post = new HttpPost(uri);
post.setEntity(builder.build());
if (log != null && log.isLoggable(Level.INFO)) {
log.info("POST: " + uri);
log.info("REQUEST JSON: " + req);
}
try (CloseableHttpResponse httpResponse = httpClient.execute(post)) {
InputStream responseStream = httpResponse.getEntity().getContent();
String response = IOUtils.toString(responseStream);
if (log != null && log.isLoggable(Level.INFO)) {
log.info("RESPONSE JSON: " + response);
PoolStats ps = cm.getTotalStats();
log.info("http-client-connections pool [avail=" + ps.getAvailable() + " leased=" + ps.getLeased() + " pending=" + ps.getPending() + "] ");
}
return new JSONObject(response);
} catch (JSONException ex) {
throw new SQLException("Invalid SDM return data: " + ex.getMessage(), ex);
} catch (IOException ex) {
throw new SQLException("Communication error with SDM host: " + ex.getMessage(), ex);
} finally {
//post.releaseConnection(); //releaseConnection and completed are the same currently in the HttpPost object.
post.completed();
}
}
public boolean doPUT(String uri, JSONObject req) throws Exception {
HttpPut put = new HttpPut(uri);
put.addHeader("Content-Type", "application/json");
StringEntity entity = new StringEntity(req.toString(), "UTF-8");
entity.setContentType("application/json");
if (log != null && log.isLoggable(Level.INFO)) {
log.info("attaching : " + req.toString());
}
put.setEntity(entity);
if (log != null && log.isLoggable(Level.INFO)) {
log.info("PUT : " + uri);
}
int statusCode;
try (CloseableHttpResponse response = httpClient.execute(put)) {
statusCode = response.getStatusLine().getStatusCode();
} finally {
put.releaseConnection();
}
return statusCode == HTTP_OK;
}
}