Binaries.java [src/csip/utils] Revision: d5970d7c0e30454d24a80dc6a55dd59e3bcbc398 Date: Sat Apr 15 07:53:13 MDT 2017
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2017, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.utils;
import csip.Config;
import static csip.Config.CSIP_DIR;
import static csip.Config.CSIP_JDBC_CHECKVALID;
import csip.Executable;
import csip.ServiceException;
import csip.SessionLogger;
import csip.annotations.Resource;
import csip.annotations.ResourceType;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.sql.DataSource;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.tomcat.jdbc.pool.DataSourceProxy;
import org.apache.tomcat.jdbc.pool.PoolProperties;
/**
*
* @author od
*/
public class Binaries {
public static final String STDOUT = "-stdout.txt";
public static final String STDERR = "-stderr.txt";
/**
* JDBC Pool class.
*/
private static class JDBCPool {
DataSource datasource;
String url;
String resolved_url = "";
Map<String, String> env;
/**
* Create a pool
* @param url
* @param env
*/
JDBCPool(String url, Map<String, String> env) {
this.url = url;
this.env = env;
}
/**
* Get the pool connection.
*
* @param logger
* @return
* @throws ServiceException
*/
Connection getConnection(SessionLogger logger) throws ServiceException {
try {
// probe for configuration change.
String res_url = Config.resolve(url);
if (datasource == null || !res_url.equals(resolved_url)) {
//Close previous datasource if it exists
if (datasource != null) {
shutdown();
}
//Remember this connection string in case it has changed.
resolved_url = res_url;
PoolProperties p = new PoolProperties();
defaultProperties(p);
p.setUrl(resolved_url);
logger.info(" JDBC pool property: url:" + resolved_url);
// load the driver if not already done do
if (!env.containsKey("driverClassName")) {
String rurl = resolved_url.toLowerCase();
if (rurl.contains(":postgresql:")) {
env.put("driverClassName", "org.postgresql.Driver");
} else if (rurl.contains(":sqlserver:")) {
env.put("driverClassName", "com.microsoft.sqlserver.jdbc.SQLServerDriver");
} else if (rurl.contains(":mysql:")) {
env.put("driverClassName", "com.mysql.jdbc.Driver");
} else if (rurl.contains(":sqlite:")) {
env.put("driverClassName", "org.sqlite.JDBC");
} else if (rurl.contains(":oracle:")) {
env.put("driverClassName", "oracle.jdbc.driver.OracleDriver");
} else if (rurl.contains(":db2:")) {
env.put("driverClassName", "com.ibm.db2.jcc.DB2Driver");
}
}
// overwrite the defaults.
for (String key : env.keySet()) {
BeanUtils.setProperty(p, key, env.get(key));
logger.info("Set JDBC pool property: " + key + ": " + env.get(key));
}
logger.info("JDBC Pool properties:" + p.toString());
datasource = new org.apache.tomcat.jdbc.pool.DataSource(p);
logger.info("Created JDBC datasource: " + datasource.toString());
}
return datasource.getConnection();
} catch (IllegalAccessException | InvocationTargetException | SQLException ex) {
logger.log(Level.SEVERE, null, ex);
throw new ServiceException("Unable to connect.. Please check the configuration parameter");
}
}
/**
* set the default properties.
*
* @param p
*/
private void defaultProperties(PoolProperties p) {
p.setDefaultAutoCommit(false);
p.setJmxEnabled(false);
p.setTestOnBorrow(true);
p.setValidationQuery("SELECT 1");
p.setTestOnReturn(false);
p.setValidationInterval(30000);
p.setMaxWait(10000);
p.setRemoveAbandonedTimeout(60);
p.setRemoveAbandoned(true);
p.setInitialSize(10);
p.setMaxActive(250);
p.setMaxIdle(100);
p.setMinIdle(10);
p.setSuspectTimeout(60);
p.setTimeBetweenEvictionRunsMillis(30000);
p.setMinEvictableIdleTimeMillis(60000);
}
/**
* Shutdown this pool.
*/
void shutdown() {
if (datasource != null && datasource instanceof DataSourceProxy) {
((DataSourceProxy) datasource).close(true);
System.out.println("Closed datasource " + datasource.toString());
}
datasource = null;
}
}
/**
* Map of JDBCPools.
*/
private static final Map<String, JDBCPool> jdbcPools = new HashMap<>();
private static final Resource[] NO_RES = new Resource[0];
private static final Object resource_lock = new Object();
private static final long DATE_AT_STARTUP = new Date().getTime();
private static final long DATE_AT_STARTUP_SEC = DATE_AT_STARTUP / 1000l;
private static SimpleCache<Class, Resource[]> rc = new SimpleCache<>();
private static SimpleCache<Class, Boolean> extractCache = new SimpleCache<>();
/**
* Shuts down all JDBC pools.
*/
public static void shutdownJDBC() {
synchronized (jdbcPools) {
for (JDBCPool pool : jdbcPools.values()) {
pool.shutdown();
}
jdbcPools.clear();
}
}
/**
* Get the architecture
* @return the architecture
*/
public static String getArch() {
String os = System.getProperty("os.name").toLowerCase();
if (os.contains("bsd")) {
os = "bsd";
} else {
os = os.substring(0, 3);
}
return os + "-" + System.getProperty("os.arch").toLowerCase();
}
/**
*
* @param p1
* @param workspace
* @param LOG
* @return
*/
private static Executable asExecutable(final Execution p1, final File workspace, final SessionLogger LOG) {
return new Executable() {
File stdout = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDOUT);
File stderr = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDERR);
Writer out;
Writer err;
@Override
public File stdout() {
return stdout;
}
@Override
public File stderr() {
return stderr;
}
@Override
public String getName() {
return p1.getFile().toString();
}
@Override
public void setArguments(Object... args) {
List<Object> l = new ArrayList<>();
Object[] o = p1.getArguments();
if (o.length > 0 && o[0] instanceof String && p1.getFile().getName().contains("wine")) {
// keep the first argument, if execution is wine.
l.add(o[0]);
}
l.addAll(Arrays.asList(args));
p1.setArguments(l);
}
@Override
public void addArguments(Object... args) {
List<Object> l = new ArrayList<>();
l.addAll(Arrays.asList(p1.getArguments()));
l.addAll(Arrays.asList(args));
p1.setArguments(l);
}
@Override
public Object[] getArguments() {
return p1.getArguments();
}
@Override
public Map<String, String> environment() {
return p1.environment();
}
@Override
public int exec() throws IOException {
p1.redirectOutput(out != null ? out : new FileWriter(stdout()));
p1.redirectError(err != null ? err : new FileWriter(stderr()));
p1.setWorkingDirectory(workspace);
p1.setLogger(LOG);
int ret = p1.exec();
if (ret != 0 && LOG.isLoggable(Level.SEVERE) && stderr().exists()) {
String s = FileUtils.readFileToString(stderr());
LOG.severe(s);
}
return ret;
}
@Override
public void redirectDefaults() throws IOException {
out = new FileWriter(stdout(), true);
err = new FileWriter(stderr(), true);
}
@Override
public void redirectOutput(String file) throws IOException {
if (file == null) {
throw new IOException("missing file name for redirect.");
}
out = new FileWriter(new File(workspace, file));
// p1.redirectOutput(new FileWriter(new File(workspace, file)));
}
@Override
public void redirectError(String file) throws IOException {
if (file == null) {
throw new IOException("missing file name for redirect.");
}
err = new FileWriter(new File(workspace, file));
// p1.redirectError(new FileWriter(new File(workspace, file)));
}
@Override
public void redirectOutput(StringWriter w) throws IOException {
out = w;
}
@Override
public void redirectError(StringWriter w) throws IOException {
err = w;
}
@Override
public void setStdoutHandler(Executable.StdHandler handler) {
p1.setStdoutHandler(handler);
}
@Override
public void setStderrHandler(Executable.StdHandler handler) {
p1.setStderrHandler(handler);
}
};
}
/**
* Get a file resources.
* @param c
* @param id
* @return
*/
public static File getResourceFile(Class<?> c, String id) throws ServiceException {
Resource resource = getResourceById(c, id);
if (resource == null) {
throw new ServiceException("No such resource: '" + id + "'");
}
String file = resource.file();
if (file == null || file.isEmpty()) {
return null;
}
File csip_home = new File(Config.getString(CSIP_DIR));
file = Config.resolve(file);
if (file.endsWith("zip") && resource.type() == ResourceType.ARCHIVE) {
return new File(csip_home, file.substring(0, file.indexOf('.')));
}
File f = new File(csip_home, file);
if (!f.exists() || !f.canRead()) {
throw new ServiceException("Cannot find file: '" + f + "'");
}
return f;
}
/**
* Get the resource as java file.
*
* @param c
* @param id
* @param workspace
* @param jars
* @param LOG
* @return
* @throws ServiceException
*/
public static Executable getResourceJava(Class<?> c, String id, final File workspace,
List<File> jars, SessionLogger LOG) throws ServiceException {
Resource r = getResourceById(c, id);
if (r == null || r.type() != ResourceType.CLASSNAME) {
throw new ServiceException("Not found: " + id);
}
String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(//r[0].args().split("\\s+"), // no jvmoptions
"-cp", asClassPath(jars),
r.file(),
r.args().split("\\s+")
);
if (r.env().length > 0) {
p.environment().putAll(parseEnv(r.env()));
}
return asExecutable(p, workspace, LOG);
}
/**
*
* @param c
* @param id
* @param workspace
* @param jars
* @param LOG
* @return
* @throws ServiceException
*/
public static Executable getResourceOMSDSL(Class<?> c, String id,
final File workspace, List<File> jars, SessionLogger LOG) throws ServiceException {
Resource r = getResourceById(c, id);
if (r == null || r.type() != ResourceType.OMS_DSL) {
throw new ServiceException("Not found: " + id);
}
String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(r.args().split("\\s+"), // jvmoptions
"-cp", asClassPath(jars),
"oms3.CLI",
"-r", r.file()
);
if (r.env().length > 0) {
p.environment().putAll(parseEnv(r.env()));
}
return asExecutable(p, workspace, LOG);
}
/**
*
* @param r
* @param options
* @param workspace
* @param jars
* @param loglevel
* @param LOG
* @return
* @throws ServiceException
*/
public static Executable getResourceOMSDSL(File r, Object[] options,
final File workspace, List<File> jars, String loglevel, SessionLogger LOG) throws ServiceException {
String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(options, // jvmoptions
"-cp", asClassPath(jars),
"oms3.CLI",
"-l", loglevel,
"-r", r.toString()
);
return asExecutable(p, workspace, LOG);
}
/**
* Get the JDBC connection from a resource definition.
*
* @param c
* @param id
* @param LOG
* @return
* @throws ServiceException
*/
public static Connection getRessourceJDBC(Class<?> c, String id, SessionLogger LOG) throws ServiceException {
synchronized (jdbcPools) {
JDBCPool p = jdbcPools.get(id);
if (p == null) {
Resource r = getResourceById(c, id);
if (r != null) {
if (r.type() == ResourceType.JDBC) {
String url = r.file();
if (url != null && !url.isEmpty()) {
Map<String, String> env = parseEnv(r.env());
p = new JDBCPool(url, env);
Connection con = p.getConnection(LOG);
try {
int valid = Config.getInt(CSIP_JDBC_CHECKVALID, -1); // default: no checking
if (valid < -1) {
valid = -1;
}
if (valid == -1 || con.isValid(valid)) {
jdbcPools.put(id, p);
return con;
} else {
throw new ServiceException("Cannot connect, invalid connection to: " + url);
}
} catch (SQLException ex) {
throw new ServiceException("Invalid timeout: " + url);
}
} else {
throw new ServiceException("No url connection string in 'file': " + id);
}
} else {
throw new ServiceException("Not a JDBC resource: " + id);
}
} else {
throw new ServiceException("No such resource: " + id);
}
}
return p.getConnection(LOG);
}
}
/**
* Add to JDBC pool
*
* @param id
* @param url
*/
public static void addToJDBCPool(String id, String url) {
jdbcPools.put(id, new JDBCPool(url, new HashMap<>()));
}
/**
* Add to JDBC pool
*
* @param id
* @param url
* @param env
*/
public static void addToJDBCPool(String id, String url, Map<String, String> env) {
jdbcPools.put(id, new JDBCPool(url, env));
}
/**
* Get Connection.
*
* @param id
* @param logger
* @return
* @throws ServiceException
*/
public static Connection getConnection(String id, SessionLogger logger) throws ServiceException {
JDBCPool p = jdbcPools.get(id);
if (p == null) {
throw new ServiceException("No such resource: " + id);
}
return p.getConnection(logger);
}
/**
* Get a file resources.
* @param c
* @param id
* @param workspace
* @param LOG
* @return
*/
public static Executable getResourceExe0(Class<?> c, String id,
final File workspace, SessionLogger LOG) throws ServiceException {
Resource resource = getResourceById(c, id);
if (resource == null) {
throw new ServiceException("Not found: " + id);
}
String file = resource.file();
if (file == null || file.isEmpty()) {
return null;
}
file = Config.resolve(file);
File executable;
if (resource.type() == ResourceType.REFERENCE) {
executable = new File(file);
// check only if there is a reference to an absolute file.
// relative commads could use the PATH.
if (executable.isAbsolute() && !executable.exists()) {
throw new ServiceException("Referenced executable not found: " + file);
}
} else if (resource.type() == ResourceType.EXECUTABLE) {
try {
File csip_home = new File(Config.getString(CSIP_DIR));
executable = unpackResource0(resource, csip_home, LOG);
} catch (IOException ex) {
throw new ServiceException(ex);
}
} else {
throw new ServiceException("Illegal resource type: " + resource.type());
}
if (!executable.canExecute()) {
executable.setExecutable(true, true);
}
Execution p = null;
List<Object> args = new ArrayList<>();
if (resource.wine() && File.pathSeparatorChar == ':') {
p = new Execution(new File(Config.getString("wine.path", "/usr/bin/wine")));
args.add(executable.toString());
} else {
p = new Execution(new File(executable.toString()));
}
if (!resource.args().isEmpty()) {
args.add(resource.args().split("\\s+"));
}
if (args.size() > 0) {
p.setArguments(args);
}
if (resource.env().length > 0) {
p.environment().putAll(parseEnv(resource.env()));
}
return asExecutable(p, workspace, LOG);
}
/**
* Get merged Resources from cache.
* @param c
* @return
*/
public static Resource[] getMergedResources(Class<?> c) {
return rc.get(c, Binaries::getMergedResources0);
}
/**
* Get merged Resources
* @param c
* @return
*/
private static Resource[] getMergedResources0(Class<?> c) {
Resource[] r = c.getAnnotationsByType(Resource.class);
if (r == null || r.length == 0) {
return NO_RES;
}
List<Resource> l = new ArrayList<>();
for (Resource res : r) {
if (res.from() != Object.class) {
Resource[] r1 = getMergedResources0(res.from());
l.addAll(Arrays.asList(r1));
} else {
l.add(res);
}
}
return l.toArray(new Resource[l.size()]);
}
/**
* Get Resource by type.
* @param c
* @param type
* @return
*/
public static Resource[] getResourcesByType(Class<?> c, ResourceType type) {
Resource[] r = getMergedResources(c);
List<Resource> l = new ArrayList<>();
for (Resource resource : r) {
if (resource.type() == type) {
l.add(resource);
}
}
return l.toArray(new Resource[l.size()]);
}
/**
* Get the resource by ID.
*
* @param c
* @param id
* @return
*/
public static Resource getResourceById(Class<?> c, String id) {
for (Resource resource : getMergedResources(c)) {
if (resource.id().equals(id)) {
return resource;
}
}
return null;
}
public static void extractAllResources(Class<?> c, SessionLogger LOG) throws ServiceException {
// do this only once per class, hence the SimpleCache
extractCache.get(c, (Class k) -> {
File csip_home = new File(Config.getString(CSIP_DIR));
csip_home.mkdirs();
for (Resource resource : getMergedResources(c)) {
if (resource.type() == ResourceType.REFERENCE
|| resource.type() == ResourceType.JDBC
|| resource.type() == ResourceType.OUTPUT
|| resource.file().isEmpty()) {
continue;
}
try {
unpackResource0(resource, csip_home, LOG);
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
return true;
});
}
/**
*
* @param name
* @param file_or_folder
* @return
* @throws IOException
* @deprecated does not unzip archives.
*/
@Deprecated()
public static File unpackResource(String name, File file_or_folder) throws IOException {
name = Config.resolve(name);
URL u = Binaries.class.getResource(name);
if (u == null) {
throw new IllegalArgumentException("No such resource " + name);
}
File outFile = null;
if (file_or_folder.isDirectory()) {
outFile = new File(file_or_folder, name);
} else {
outFile = file_or_folder;
}
synchronized (resource_lock) {
// newer date?
if (outFile.exists()
&& (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l))) {
return outFile;
}
if (!outFile.getParentFile().exists()) {
outFile.getParentFile().mkdirs();
}
InputStream in = new BufferedInputStream(u.openStream());
FileUtils.copyInputStreamToFile(in, outFile);
in.close();
outFile.setExecutable(true);
outFile.setReadable(true);
outFile.setLastModified(DATE_AT_STARTUP);
return outFile;
}
}
/**
* This is needed because of services calling the above method.
* @param r
* @param LOG
* @param file_or_folder
* @return
* @throws IOException
*/
public static File unpackResource0(Resource r, File file_or_folder, SessionLogger LOG) throws IOException {
String name = r.file();
name = Config.resolve(name);
LOG.info("accessing resource : " + name);
File outFile = file_or_folder;
if (file_or_folder.isDirectory()) {
outFile = new File(file_or_folder, name);
}
synchronized (resource_lock) {
// check by date, ignore milliseconds since they might be not stored.
if (outFile.exists() && (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l))) {
return outFile;
}
if (!outFile.getParentFile().exists()) {
outFile.getParentFile().mkdirs();
}
URL u = Binaries.class.getResource(name);
if (u == null) {
throw new IllegalArgumentException("No such resource " + name);
}
InputStream in = new BufferedInputStream(u.openStream());
FileUtils.copyInputStreamToFile(in, outFile);
in.close();
if (r.type() == ResourceType.EXECUTABLE) {
if (!outFile.canExecute()) {
outFile.setExecutable(true, true);
}
}
outFile.setReadable(true);
outFile.setLastModified(DATE_AT_STARTUP);
LOG.info("extracted: " + name + " to " + outFile);
// file was an archive, unzip it.
if (name.endsWith(".zip") && (r.type() == ResourceType.ARCHIVE)) {
ZipFiles.unzip(outFile);
LOG.info("unzipped: " + outFile);
}
return outFile;
}
}
/**
*
* @param name
* @param newName
* @return
* @throws IOException
*/
@Deprecated
public static File unpackResourceAbsolute(String name, String newName) throws IOException {
name = Config.resolve(name);
URL u = Binaries.class.getResource(name);
if (u == null) {
throw new IllegalArgumentException("No such resource " + name);
}
File outFile = new File(newName);
synchronized (resource_lock) {
if (outFile.exists()
&& (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l))) {
// the local file exists and is newer.
return outFile;
}
if (!outFile.getParentFile().exists()) {
outFile.getParentFile().mkdirs();
}
InputStream in = new BufferedInputStream(u.openStream());
FileUtils.copyInputStreamToFile(in, outFile);
in.close();
outFile.setExecutable(true);
outFile.setLastModified(DATE_AT_STARTUP);
return outFile;
}
}
/**
* get the FS usage of that directory/file location in percent.
*
* @param file
* @return value 0.0 .. 1.0, 1.0 means 100% full.
*/
public static double getFSUsage(File file) {
return 1 - (double) file.getUsableSpace() / file.getTotalSpace();
}
/**
* get the library jars
*
* @param dir
* @return
*/
public static List<File> getJars(File dir) {
File[] f = dir.listFiles((FilenameFilter) new WildcardFileFilter("*.jar"));
if (f == null) {
return new ArrayList<>();
}
return Arrays.asList(f);
}
/**
* Convert the files to a classpath
*
* @param f
* @return
*/
public static String asClassPath(List<File> f) {
StringBuilder b = new StringBuilder();
for (int i = 0; i < f.size(); i++) {
if (f.get(i).getName().endsWith(".jar")) {
b.append(f.get(i).toString());
if (i < f.size() - 1) {
b.append(File.pathSeparatorChar);
}
}
}
return b.toString();
}
/**
* Converts a map to system properties.
*
* @param m
* @return
*/
public static String[] asSysProps(Map<String, String> m) {
List<String> b = new ArrayList<>();
for (String key : m.keySet()) {
String val = m.get(key);
b.add("-D" + key + "=" + val);
}
return b.toArray(new String[b.size()]);
}
/**
*
* @param env
* @return
*/
public static Map<String, String> parseEnv(String[] env) {
Map<String, String> m = new HashMap<>();
if (env == null) {
return m;
}
for (String s : env) {
if (s == null || s.isEmpty()) {
continue;
}
String t[] = s.trim().split("\\s*=\\s*");
if (t.length == 2) {
m.put(t[0], Config.resolve(t[1]));
}
}
return m;
}
/**
* Parses a string into a number of bytes. e.g. "1KB" = 1024,
* @param in
* @return the size in bytes.
*/
public static long parseByteSize(String in) {
in = in.trim().replaceAll(",", ".");
try {
return Long.parseLong(in);
} catch (NumberFormatException e) {
}
Matcher m = Pattern.compile("([\\d.,]+)\\s*(\\w)").matcher(in);
m.find();
double scale = 1;
switch (m.group(2).charAt(0)) {
case 'G':
case 'g':
scale *= 1024;
case 'M':
case 'm':
scale *= 1024;
case 'K':
case 'k':
scale *= 1024;
break;
default:
throw new IllegalArgumentException(in);
}
return Math.round(Double.parseDouble(m.group(1)) * scale);
}
// public static void main(String[] args) {
// String a = Config.resolve("${csip.arch} all to ${user.dir} is true. '${xqy }'live in ${csip.dir} ttl is ${csip.session.ttl.cancelled} ....");
// System.out.println(a);
////
// }
// String a = "jdbc:sqlserver://129.82.20.242:1433;databaseName=lmod_ZedX;user=lmod-rw;password=managements";
// String b = a.replaceAll("password=.+[;]?", "password=****;");
//
// System.out.println(b);
// System.out.println(a);
//
//// File dir = new File("/tmp");
////
//// String b = "odd/bcd/ab.txt";
// File f = new File("odd/bcd/ab.txt");
// f.getParentFile().mkdirs();
//
// FileOutputStream os = new FileOutputStream(f);
// os.write("olaf".getBytes());
// os.close();
//// IOUtils.writeStringToFile(f, "hi there");
}