Utils.java [src/csip] Revision: default Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip;
import csip.api.server.ServiceException;
import csip.annotations.Author;
import csip.annotations.Description;
import csip.annotations.Documentation;
import csip.annotations.License;
import csip.annotations.State;
import csip.annotations.Name;
import csip.annotations.VersionInfo;
import csip.utils.Services;
import java.io.BufferedInputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Level;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Path;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.UriBuilder;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
/**
* CSIP core package Utilities.
*
* @author od
*/
public class Utils {
static final void checkRemoteAccessACL(HttpServletRequest req) {
String reqIp = req.getHeader("X-Forwarded-For");
if (reqIp == null)
reqIp = req.getRemoteAddr();
if (!checkRemoteAccessACL(reqIp)) {
Config.LOG.log(Level.WARNING, req.getMethod() + " " + req.getRequestURI() + ", denied for " + reqIp);
throw new WebApplicationException(Response.Status.UNAUTHORIZED);
}
Config.LOG.log(Level.INFO, req.getMethod() + " " + req.getRequestURI() + ", OK for " + reqIp);
}
static final boolean checkRemoteAccessACL(String ip) {
String acls = Config.getString(Config.CSIP_REMOTE_ACL);
String[] acl = acls.split("\\s+");
for (String ace : acl) {
try {
if (new IpMatcher(ace).matches(ip))
return true;
} catch (UnknownHostException E) {
Config.LOG.log(Level.WARNING, E.getMessage(), E);
return false;
}
}
return false;
}
/**
* Resolve a string with system and CSIP properties.
*
* @param str the string to resolve
* @return the resolved string.
*/
public static String resolve(String str) {
if (str == null)
return null;
if (!str.contains("${"))
return str;
String res = resolve0(str, Config.getMergedProperties(), new HashSet<>());
if (res.contains("${"))
Config.LOG.warning("Resolving one or more varariables failed in: " + res);
return res;
}
/**
* property substitution in a string.
*
* @param str
* @return
*/
private static String resolve0(String str, Properties prop, Set<String> keys) {
int idx = 0;
while (idx < str.length()) {
int start = str.indexOf("${", idx);
int end = str.indexOf("}", idx);
if (start == -1 || end == -1 || end < start)
break;
String key = str.substring(start + 2, end);
if (keys.contains(key)) {
System.err.println("Circular property reference: " + key);
break;
}
String val = prop.getProperty(key);
if (val != null) {
keys.add(key);
val = resolve0(val, prop, keys);
keys.remove(key);
str = str.replace("${" + key + "}", val);
idx = start + val.length();
} else {
idx = start + key.length() + 3;
}
}
return str;
}
static void callStaticMethodIfExist(Class<?> c, String method) {
try {
Method m = c.getMethod(method);
m.invoke(null);
Config.LOG.info(" Invoked '" + method + "' in: " + c);
} catch (NoSuchMethodException ex) {
return; // no problem
} catch (SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
Config.LOG.log(Level.SEVERE, null, ex);
}
}
static String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit)
return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
static String getServicePath(Class<?> c) {
Path p = c.getAnnotation(Path.class);
return (p != null) ? p.value() : "";
}
static String getServiceName(Class<?> c) {
Name p = c.getAnnotation(Name.class);
return (p != null) ? p.value() : c.getName();
}
/**
* Check if a class is a CSIP service. (formData/d/or MDS subclass)
*
* @param s the class to check,
* @return
*/
public static boolean isCsipService(Class<?> s) {
// Must be ModelDataservice sub class.
if (!ModelDataService.class.isAssignableFrom(s))
return false;
// check service path, must start with 'formData' or 'd' or 'p'
Path p = s.getAnnotation(Path.class);
if (p == null)
return false;
if (p.value().startsWith("m/") // model service
|| p.value().startsWith("p/") // platform service
|| p.value().startsWith("d/")) // data service
return true;
return false;
}
public static String[] getURIParts(String uri) throws URISyntaxException {
URI b = new URI(uri);
String[] path = b.getPath().split("/");
String[] p = new String[2 + path.length];
p[0] = b.getScheme() + "://";
p[1] = b.getHost();
p[2] = b.getPort() == -1 ? "" : (":" + Integer.toString(b.getPort()));
for (int i = 1; i < path.length; i++) {
p[i + 2] = path[i];
}
return p;
}
public static String replaceHostinURI(URI uri, String newHost) throws Exception {
int port = Config.getInt("csip.peer.port", 8080);
UriBuilder b = UriBuilder.fromUri(uri);
URI u = b.host(newHost).port(port).build();
return u.toString();
}
public static String getPublicRequestURL(HttpServletRequest httpReq) {
String requrl = httpReq.getRequestURL().toString();
String p = httpReq.getHeader("X-Forwarded-Proto"); // https?
if (p != null) {
UriBuilder b = UriBuilder.fromUri(requrl);
requrl = b.scheme(p).build().toString();
}
String host = toPublicURL(requrl).toString();
return host;
}
public static URI toPublicURL(String u) {
UriBuilder b = UriBuilder.fromUri(u);
String s = Config.getString("csip.public.scheme");
if (s != null)
b = b.scheme(s);
s = Config.getString("csip.public.host");
if (s != null)
b = b.host(s);
s = Config.getString("csip.public.port");
if (s != null)
b = b.port(Integer.parseInt(s));
s = Config.getString("csip.public.contextprefix");
if (s != null) {
//Find the third '/' because http://google.com:3022/<context> should basically always be 3 slashes in.
String bUrl = b.build().toString();
int indexOfThirdSlash = StringUtils.ordinalIndexOf(bUrl, "/", 3);
//Split the uri on that slash
String hostAndPort = bUrl.substring(0, indexOfThirdSlash);
String remainder = bUrl.substring(indexOfThirdSlash);
//Put the context prefix in the middle.
String newBUrl = hostAndPort + "/" + s + remainder;
//Read back into b as a URI so we don't lose changes.
b = UriBuilder.fromUri(newBUrl);
}
return b.build();
}
/**
* Maps the incoming URL to an internal one.
*
* @param incoming the original, incoming url.
* @param path the new path (another service)
* @return
* @throws MalformedURLException
*/
public static URI toInternalURL(String incoming, String path) throws MalformedURLException {
UriBuilder internal = UriBuilder.fromUri(incoming);
if (path != null)
internal = internal.replacePath(path);
String uri = Config.getString("csip.internal.uri");
if (uri == null)
return internal.build();
URL url = new URL(uri);
if (url.getPort() != -1)
internal = internal.port(url.getPort());
if (url.getHost() != null)
internal = internal.host(url.getHost());
if (url.getProtocol() != null)
internal = internal.scheme(url.getProtocol());
return internal.build();
}
/**
* Populate service matainfo into a JSONObject
*
* @return The service name or null if there is none
*/
public static JSONObject getServiceInfo(Class<?> c) throws JSONException {
JSONObject o = new JSONObject();
Name p = c.getAnnotation(Name.class);
if (p != null) {
o.put("name", p.value());
}
Description d = c.getAnnotation(Description.class);
if (d != null) {
o.put("description", d.value());
}
Documentation doc = c.getAnnotation(Documentation.class);
if (doc != null)
o.put("documentation", doc.value());
License lic = c.getAnnotation(License.class);
if (lic != null)
o.put("license", lic.value());
VersionInfo ver = c.getAnnotation(VersionInfo.class);
if (ver != null)
o.put("version", ver.value());
Author aut = c.getAnnotation(Author.class);
if (aut != null)
o.put("author", (aut.name() + " " + aut.email() + " " + aut.org()).trim());
Deprecated dep = c.getAnnotation(Deprecated.class);
if (dep != null) {
o.put("state", State.DEPRECATED);
} else {
State sta = c.getAnnotation(State.class);
if (sta != null) {
o.put("state", sta.value());
}
}
o.put("built", Config.getFullVersion());
return o;
}
public static File[] expandFiles(File work, String pattern) throws IOException {
if (pattern == null || pattern.isEmpty())
return new File[]{};
List<File> f = new ArrayList<>();
final PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + work + "/" + pattern);
Files.walkFileTree(Paths.get(work.toString()), new SimpleFileVisitor<java.nio.file.Path>() {
@Override
public FileVisitResult postVisitDirectory(java.nio.file.Path dir, IOException exc) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) throws IOException {
if (matcher.matches(file)) {
f.add(file.toFile());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(java.nio.file.Path file, IOException exc) throws IOException {
return FileVisitResult.CONTINUE;
}
});
return f.toArray(new File[0]);
}
/**
* Copy the formParameter (files) to a directory, if the file is an archive,
* The content will be extracted.
*
* @param log
* @param dir
* @param forms
* @param unpack
* @return the extracted files.
* @throws ServiceException
*/
public static String[] copyAndExtract(SessionLogger log, File dir,
Map<String, Services.FormDataParameter> forms, boolean unpack) throws ServiceException {
List<String> files = new ArrayList<>();
if (!dir.exists())
dir.mkdirs();
for (Services.FormDataParameter fd : forms.values()) {
if (!fd.isFile())
continue;
String name = fd.getFilename();
name = name.replace('\\', '/');
InputStream fis = fd.getInputStream();
String lcName = name.toLowerCase();
// archives
try {
if ((lcName.endsWith(".bz2") || lcName.endsWith(".gz")) && unpack) {
// wrapper (supports single files, as well as tar.gz / tar.bzls
fis = new CompressorStreamFactory().createCompressorInputStream(new BufferedInputStream(fis));
name = removeExt(name);
lcName = name.toLowerCase();
}
if ((lcName.endsWith(".zip") || lcName.endsWith(".tar")) && unpack) {
ArchiveInputStream is = new ArchiveStreamFactory().createArchiveInputStream(new BufferedInputStream(fis));
ArchiveEntry entry = null;
while ((entry = is.getNextEntry()) != null) {
if (is.canReadEntryData(entry)) {
if (entry.isDirectory()) {
new File(dir, entry.getName()).mkdirs();
} else {
File f = new File(dir, entry.getName());
if (!f.getParentFile().exists())
f.getParentFile().mkdirs();
try (FileOutputStream ous = new FileOutputStream(f)) {
IOUtils.copy(is, ous);
}
files.add(entry.getName());
f.setLastModified(entry.getLastModifiedDate().getTime());
log.info("Extracted :" + entry.getName() + " as " + f);
}
}
}
is.close();
} else {
File f = new File(dir, name);
if (!f.getParentFile().exists())
f.getParentFile().mkdirs();
try (FileOutputStream ous = new FileOutputStream(f)) {
IOUtils.copy(fis, ous);
}
files.add(name);
if (log.isLoggable(Level.INFO)) {
log.info("copy form data file: " + name + " to " + dir);
}
}
} catch (CompressorException | ArchiveException | IOException ex) {
throw new ServiceException(ex);
}
try {
fis.close();
} catch (IOException ex) {
log.log(Level.SEVERE, "", ex);
}
}
return files.toArray(new String[files.size()]);
}
public static String removeExt(String name) {
return name.substring(0, name.lastIndexOf("."));
}
public static String removeFirstLastChar(String text) {
return text.substring(1, text.length() - 1);
}
}