Binaries.java [src/csip/utils] Revision: Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.utils;
import csip.Config;
import static csip.Config.CSIP_DIR;
import static csip.Config.CSIP_JDBC_CHECKVALID;
import csip.api.server.Executable;
import csip.api.server.ServiceException;
import csip.SessionLogger;
import csip.Utils;
import csip.annotations.Resource;
import csip.annotations.ResourceType;
import static csip.annotations.ResourceType.CLASSNAME;
import static csip.annotations.ResourceType.JAR;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException;
import java.net.URL;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.sql.DataSource;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.tomcat.jdbc.pool.DataSourceProxy;
import org.apache.tomcat.jdbc.pool.PoolProperties;
/**
*
* @author od
*/
public class Binaries {
public static final String STDOUT = "-stdout.txt";
public static final String STDERR = "-stderr.txt";
public static final String AUTO_RUN = "auto";
public static final String DRIVER = "driverClassName";
/**
* JDBC Pool class.
*/
private static class JDBCPool {
DataSource datasource;
String url;
String resolved_url = "";
Map<String, String> env;
/**
* Create a pool
* @param url
* @param env
*/
JDBCPool(String url, Map<String, String> env) {
this.url = url;
this.env = env;
}
/**
* Get the pool connection.
*
* @param logger
* @return
* @throws ServiceException
*/
Connection getConnection(SessionLogger logger) throws ServiceException {
try {
// probe for configuration change.
String res_url = Utils.resolve(url);
if (datasource == null || !res_url.equals(resolved_url)) {
//Close previous datasource if it exists
if (datasource != null)
shutdown();
//Remember this connection string in case it has changed.
resolved_url = res_url;
PoolProperties p = new PoolProperties();
defaultProperties(p);
p.setUrl(resolved_url);
logger.info(" JDBC pool property: url:" + resolved_url);
// load the driver if not already done do
if (!env.containsKey(DRIVER)) {
String rurl = resolved_url.toLowerCase();
if (rurl.contains(":postgresql:"))
env.put(DRIVER, "org.postgresql.Driver");
else if (rurl.contains(":sqlserver:"))
env.put(DRIVER, "com.microsoft.sqlserver.jdbc.SQLServerDriver");
else if (rurl.contains(":mysql:"))
env.put(DRIVER, "com.mysql.jdbc.Driver");
else if (rurl.contains(":sqlite:"))
env.put(DRIVER, "org.sqlite.JDBC");
else if (rurl.contains(":oracle:"))
env.put(DRIVER, "oracle.jdbc.driver.OracleDriver");
else if (rurl.contains(":db2:"))
env.put(DRIVER, "com.ibm.db2.jcc.DB2Driver");
else if (rurl.contains(":sdm:"))
env.put(DRIVER, "csip.sdm.SDMDriver");
}
// overwrite the defaults.
Properties dbp = new Properties();
for (String key : env.keySet()) {
String val = env.get(key);
try {
BeanUtils.setProperty(p, key, val);
logger.info("Set JDBC pool property: " + key + ": " + val);
} catch (IllegalAccessException | InvocationTargetException E) {
dbp.setProperty(key, val);
logger.info("Set DB property: " + key + ": " + val);
}
}
if (!dbp.isEmpty())
p.setDbProperties(dbp);
logger.info("JDBC Pool properties:" + p.toString());
datasource = new org.apache.tomcat.jdbc.pool.DataSource(p);
logger.info("Created JDBC datasource: " + datasource.toString());
}
return datasource.getConnection();
} catch (SQLException ex) {
logger.log(Level.SEVERE, null, ex);
throw new ServiceException("Unable to connect.. Please check the configuration parameter");
}
}
/**
* set the default properties.
*
* @param p
*/
private void defaultProperties(PoolProperties p) {
p.setDefaultAutoCommit(false);
p.setJmxEnabled(false);
p.setTestOnBorrow(true);
p.setValidationQuery("SELECT 1");
p.setTestOnReturn(false);
p.setValidationInterval(30000);
p.setMaxWait(10000);
p.setRemoveAbandonedTimeout(60);
p.setRemoveAbandoned(true);
p.setInitialSize(10);
p.setMaxActive(250);
p.setMaxIdle(100);
p.setMinIdle(10);
p.setSuspectTimeout(60);
p.setTimeBetweenEvictionRunsMillis(30000);
p.setMinEvictableIdleTimeMillis(60000);
}
/**
* Shutdown this pool.
*/
void shutdown() {
if (datasource instanceof DataSourceProxy) {
((DataSourceProxy) datasource).close(true);
System.out.println("Closed datasource " + datasource.toString());
}
datasource = null;
}
}
/**
* Map of JDBCPools.
*/
private static final Map<String, JDBCPool> jdbcPools = new HashMap<>();
private static final List<Resource> NO_RES = new ArrayList<>();
private static final Object resource_lock = new Object();
private static final long DATE_AT_STARTUP = new Date().getTime();
private static final long DATE_AT_STARTUP_SEC = DATE_AT_STARTUP / 1000l;
private static SimpleCache<Class<?>, List<Resource>> rc = new SimpleCache<>();
private static SimpleCache<Class<?>, Boolean> extractCache = new SimpleCache<>();
static boolean newCP = Config.getBoolean("csip.newcp", false);
/**
* Shuts down all JDBC pools.
*/
public static void shutdownJDBC() {
if (newCP) {
ConnectionPools.singleton().shutdownJDBC();
return;
}
synchronized (jdbcPools) {
for (JDBCPool pool : jdbcPools.values()) {
pool.shutdown();
}
jdbcPools.clear();
}
}
/**
* Get the architecture
* @return the architecture
*/
public static String getArch() {
String os = System.getProperty("os.name").toLowerCase();
if (os.contains("bsd"))
os = "bsd";
else
os = os.substring(0, 3);
return os + "-" + System.getProperty("os.arch").toLowerCase();
}
/**
*
* @param p1
* @param workspace
* @param LOG
* @return
*/
private static Executable asExecutable(final Execution p1, final File workspace, final SessionLogger LOG) {
return new Executable() {
File stdout = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDOUT);
File stderr = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDERR);
Writer out;
Writer err;
@Override
public File stdout() {
return stdout;
}
@Override
public File stderr() {
return stderr;
}
@Override
public String getName() {
return p1.getFile().toString();
}
@Override
public void setArguments(Object... args) {
List<Object> l = new ArrayList<>();
Object[] o = p1.getArguments();
if (o.length > 0 && o[0] instanceof String && p1.getFile().getName().contains("wine"))
// keep the first argument, if execution is wine.
l.add(o[0]);
l.addAll(Arrays.asList(args));
p1.setArguments(l);
}
@Override
public void addArguments(Object... args) {
List<Object> l = new ArrayList<>();
l.addAll(Arrays.asList(p1.getArguments()));
l.addAll(Arrays.asList(args));
p1.setArguments(l);
}
@Override
public Object[] getArguments() {
return p1.getArguments();
}
@Override
public Map<String, String> environment() {
return p1.environment();
}
@Override
public int exec() throws IOException {
p1.redirectOutput(out != null ? out : new FileWriter(stdout()));
p1.redirectError(err != null ? err : new FileWriter(stderr()));
p1.setWorkingDirectory(workspace);
p1.setLogger(LOG);
int ret = p1.exec();
if (ret != 0 && LOG.isLoggable(Level.SEVERE) && stderr().exists()) {
String s = FileUtils.readFileToString(stderr(), "UTF-8");
LOG.severe(s);
}
return ret;
}
@Override
public void redirectDefaults() throws IOException {
out = new FileWriter(stdout(), true);
err = new FileWriter(stderr(), true);
}
@Override
public void redirectOutput(String file) throws IOException {
if (file == null)
throw new IOException("missing file name for redirect.");
out = new FileWriter(new File(workspace, file));
}
@Override
public void redirectError(String file) throws IOException {
if (file == null)
throw new IOException("missing file name for redirect.");
err = new FileWriter(new File(workspace, file));
}
@Override
public void redirectOutput(StringWriter w) throws IOException {
out = w;
}
@Override
public void redirectError(StringWriter w) throws IOException {
err = w;
}
@Override
public void setStdoutHandler(Executable.StdHandler handler) {
p1.setStdoutHandler(handler);
}
@Override
public void setStderrHandler(Executable.StdHandler handler) {
p1.setStderrHandler(handler);
}
@Override
public void setTimeout(long time, TimeUnit unit) {
p1.setTimeout(time, unit);
}
};
}
/**
*
* @param c
* @param id
* @return
* @throws ServiceException
*/
public static Resource getResource(Class<?> c, String id) throws ServiceException {
return getResourceById(c, id);
}
/**
* Get a file resources.
* @param c the service class
* @param id the Resource ID
* @return the file.
* @throws csip.api.server.ServiceException
*/
public static File getResourceFile(Class<?> c, String id) throws ServiceException {
Resource resource = getResourceById(c, id);
if (resource == null)
throw new ServiceException("No such resource: '" + id + "'");
String file = resource.file();
if (file == null || file.isEmpty())
return null;
file = Utils.resolve(file);
if (resource.type() == ResourceType.REFERENCE) {
File f = new File(file);
if (!f.exists() || !f.canRead())
throw new ServiceException("Cannot find file: '" + f + "'");
return f;
}
File csip_home = new File(Config.getString(CSIP_DIR));
if (file.endsWith("zip") && resource.type() == ResourceType.ARCHIVE)
return new File(csip_home, file.substring(0, file.indexOf('.')));
File f = new File(csip_home, file);
if (!f.exists() || !f.canRead())
throw new ServiceException("Cannot find file: '" + f + "'");
return f;
}
/**
* Get the resource as java file.
*
* @param c the service class
* @param id the resource id
* @param workspace the session workspace
* @param jars the jar files
* @param LOG Session Logger
* @return the Executable
* @throws ServiceException
*/
public static Executable getResourceJava(Class<?> c, String id, final File workspace,
List<File> jars, SessionLogger LOG) throws ServiceException {
Resource r = getResourceById(c, id);
if (r == null || r.type() != ResourceType.CLASSNAME)
throw new ServiceException("Not found: " + id);
return getResourceJava(r, workspace, jars, LOG);
}
public static Executable getResourceJava(Resource r, final File workspace,
List<File> jars, SessionLogger LOG) throws ServiceException {
String oms_java_home = Config.getString("java.home", "/opt/jdk1.7.0_45");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(//r[0].args().split("\\s+"), // no jvmoptions
"-cp", asClassPath(jars),
r.file(),
r.args().split("\\s+")
);
if (r.env().length > 0)
p.environment().putAll(parseEnv(r.env()));
return asExecutable(p, workspace, LOG);
}
public static Executable getResourceJava0(Resource r, File workspace,
SessionLogger LOG) throws ServiceException {
if (r.type() != CLASSNAME && r.type() != JAR)
throw new IllegalArgumentException("Invalid java resource type");
String oms_java_home = Config.getString("java.home");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(
r.type() == JAR ? "-jar" : "",
Utils.resolve(r.file()),
r.args().split("\\s+")
);
if (r.env().length > 0)
p.environment().putAll(parseEnv(r.env()));
return asExecutable(p, workspace, LOG);
}
static String adjustPathSeparator(String s) {
return s.replace(';', File.pathSeparatorChar).replace(':', File.pathSeparatorChar);
}
private static Executable getScriptResource(Resource r,
SessionLogger LOG, Class<?> service, File workspace,
String interpreter, String ext, ResourceType t) throws ServiceException {
Execution p = new Execution(new File(interpreter));
String file = r.file();
if (file == null || file.isEmpty()) {
if (service == null)
return null;
file = '/' + service.getName().replace('.', '/') + ext;
}
file = Utils.resolve(file);
File f = new File(Config.getString(CSIP_DIR), file);
File csip_home = new File(Config.getString(CSIP_DIR));
try {
unpackResource0(file, t, csip_home, LOG);
} catch (IOException E) {
throw new ServiceException(E);
}
// try again....
if (!f.exists() || !f.canRead())
throw new ServiceException("Cannot find file: '" + f + "'");
p.setArguments(f.toString(), r.args().split("\\s+"));
if (r.env().length > 0)
p.environment().putAll(parseEnv(r.env()));
return asExecutable(p, workspace, LOG);
}
public static Executable getResourcePython(Resource r, final File workspace,
SessionLogger LOG, Class<?> service) throws ServiceException {
return getScriptResource(r, LOG, service, workspace,
r.type() == ResourceType.PYTHON3
? Config.getString("csip.python3.path", "python3")
: Config.getString("csip.python.path", "python"), ".py", ResourceType.PYTHON3);
}
public static Executable getResourceRScript(Resource r, final File workspace,
SessionLogger LOG, Class<?> service) throws ServiceException {
return getScriptResource(r, LOG, service, workspace,
Config.getString("csip.rscript.path", "Rscript"), ".r", ResourceType.RSCRIPT);
}
public static Executable getResourceBash(Resource r, final File workspace,
SessionLogger LOG, Class<?> service) throws ServiceException {
return getScriptResource(r, LOG, service, workspace,
Config.getString("csip.bash.path", "bash"), ".sh", ResourceType.RSCRIPT);
}
// public static Executable getResourcePython0(Resource r, final File workspace,
// SessionLogger LOG, Class<?> service) throws ServiceException {
// Execution p = new Execution(new File(r.type() == ResourceType.PYTHON3
// ? Config.getString("csip.python3.path", "python3")
// : Config.getString("csip.python.path", "python")));
//
// String file = r.file();
// if (file == null || file.isEmpty()) {
// if (service == null) {
// return null;
// }
// file = '/' + service.getName().replace('.', '/') + ".py";
// }
// file = Utils.resolve(file);
// File f = new File(Config.getString(CSIP_DIR), file);
// File csip_home = new File(Config.getString(CSIP_DIR));
// try {
// unpackResource0(file, ResourceType.PYTHON3, csip_home, LOG);
// } catch (IOException E) {
// throw new ServiceException(E);
// }
// // try again....
// if (!f.exists() || !f.canRead()) {
// throw new ServiceException("Cannot find file: '" + f + "'");
// }
// p.setArguments(f.toString(), r.args().split("\\s+"));
// if (r.env().length > 0) {
// p.environment().putAll(parseEnv(r.env()));
// }
// return asExecutable(p, workspace, LOG);
// }
//
//
// public static Executable getResourceRScript(Resource r, final File workspace,
// SessionLogger LOG, Class<?> service) throws ServiceException {
// Execution p = new Execution(new File(Config.getString("csip.rscript.path", "rscript")));
//
// String file = r.file();
// if (file == null || file.isEmpty()) {
// if (service == null) {
// return null;
// }
// file = '/' + service.getName().replace('.', '/') + ".r";
// }
// file = Utils.resolve(file);
// File f = new File(Config.getString(CSIP_DIR), file);
// File csip_home = new File(Config.getString(CSIP_DIR));
// try {
// unpackResource0(file, ResourceType.RSCRIPT, csip_home, LOG);
// } catch (IOException E) {
// throw new ServiceException(E);
// }
// // try again....
// if (!f.exists() || !f.canRead()) {
// throw new ServiceException("Cannot find file: '" + f + "'");
// }
// p.setArguments(f.toString(), r.args().split("\\s+"));
// if (r.env().length > 0) {
// p.environment().putAll(parseEnv(r.env()));
// }
// return asExecutable(p, workspace, LOG);
// }
// /**
// * Get the resource as DSL.
// *
// * @param c the service class
// * @param id the resource id
// * @param workspace the workspace
// * @param jars the jar files
// * @param LOG logger
// * @return the Executable interface
// * @throws ServiceException
// */
// public static Executable getResourceOMSDSL(Class<?> c, String id,
// final File workspace, List<File> jars, SessionLogger LOG) throws ServiceException {
// Resource r = getResourceById(c, id);
// if (r == null || r.type() != ResourceType.OMS_DSL)
// throw new ServiceException("Not found: " + id);
//
// return getResourceOMSDSL(r, workspace, jars, LOG);
// }
//
//
// public static Executable getResourceOMSDSL(Resource r,
// final File workspace, List<File> jars, SessionLogger LOG) throws ServiceException {
// String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
// Execution p = new Execution(new File(oms_java_home, "/bin/java"));
// p.setArguments(r.args().split("\\s+"), // jvmoptions
// "-cp", asClassPath(jars),
// "oms3.CLI",
// "-r", r.file()
// );
// if (r.env().length > 0)
// p.environment().putAll(parseEnv(r.env()));
//
// return asExecutable(p, workspace, LOG);
// }
/**
*
* @param r
* @param options
* @param workspace
* @param jars
* @param loglevel
* @param LOG
* @return
* @throws ServiceException
*/
public static Executable getResourceOMSDSL(File r, Object[] options,
final File workspace, List<File> jars, String loglevel, SessionLogger LOG) throws ServiceException {
String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
Execution p = new Execution(new File(oms_java_home, "/bin/java"));
p.setArguments(options, // jvmoptions
"-cp", asClassPath(jars),
"oms3.CLI",
"-l", loglevel,
"-r", r.toString()
);
return asExecutable(p, workspace, LOG);
}
/**
* Get the JDBC connection from a resource definition.
*
* @param c
* @param id
* @param LOG
* @return
* @throws ServiceException
*/
public static Connection getResourceJDBC(Class<?> c, String id, SessionLogger LOG) throws ServiceException {
if (newCP)
return ConnectionPools.singleton().getResourceJDBC(c, id, LOG);
synchronized (jdbcPools) {
JDBCPool p = jdbcPools.get(id);
if (p == null) {
Resource r = getResourceById(c, id);
if (r != null) {
if (r.type() == ResourceType.JDBC) {
String url = r.file();
if (url != null && !url.isEmpty()) {
Map<String, String> env = parseEnv(r.env());
p = new JDBCPool(url, env);
Connection con = p.getConnection(LOG);
try {
int valid = Config.getInt(CSIP_JDBC_CHECKVALID, -1); // default: no checking
if (valid < -1)
valid = -1;
if (valid == -1 || con.isValid(valid)) {
jdbcPools.put(id, p);
return con;
} else {
throw new ServiceException("Cannot connect, invalid connection to: " + url);
}
} catch (SQLException ex) {
throw new ServiceException("Invalid timeout: " + url);
}
} else {
throw new ServiceException("No url connection string in 'file': " + id);
}
} else {
throw new ServiceException("Not a JDBC resource: " + id);
}
} else {
throw new ServiceException("No such resource: " + id);
}
}
return p.getConnection(LOG);
}
}
/**
* Add to JDBC pool
*
* @param id
* @param url
*/
public static void addToJDBCPool(String id, String url) {
if (newCP)
ConnectionPools.singleton().addToJDBCPool(id, url);
else
jdbcPools.put(id, new JDBCPool(url, new HashMap<>()));
}
/**
* Add to JDBC pool
*
* @param id
* @param url
* @param env
*/
public static void addToJDBCPool(String id, String url, Map<String, String> env) {
if (newCP)
ConnectionPools.singleton().addToJDBCPool(id, url, env);
else
jdbcPools.put(id, new JDBCPool(url, env));
}
/**
* Get Connection.
*
* @param id
* @param logger
* @return
* @throws ServiceException
*/
public static Connection getConnection(String id, SessionLogger logger) throws ServiceException {
if (newCP)
return ConnectionPools.singleton().getConnection(id, logger);
JDBCPool p = jdbcPools.get(id);
if (p == null)
throw new ServiceException("No such resource: " + id);
return p.getConnection(logger);
}
public static void doInstall(Resource resource, SessionLogger LOG) throws IOException {
String file = resource.file();
if (file.isEmpty())
return;
file = Utils.resolve(file);
File csip_home = new File(Config.getString(CSIP_DIR));
File outFile = new File(csip_home, file);
synchronized (resource_lock) {
// check by date, ignore milliseconds since they might be not stored.
if (outFile.exists() && (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l)))
return;
File executable = unpackResource0(resource, csip_home, LOG);
if (!executable.canExecute())
executable.setExecutable(true, true);
Execution p = null;
List<Object> args = new ArrayList<>();
if (resource.wine() && File.pathSeparatorChar == ':') {
p = new Execution(new File(Config.getString("wine.path", "/usr/bin/wine")));
args.add(executable.toString());
} else {
p = new Execution(new File(executable.toString()));
}
if (!resource.args().isEmpty())
args.add(resource.args().split("\\s+"));
if (args.size() > 0)
p.setArguments(args);
if (resource.env().length > 0)
p.environment().putAll(parseEnv(resource.env()));
LOG.info("executing install : " + outFile);
Executable ex = asExecutable(p, outFile.getParentFile(), LOG);
int ret = ex.exec();
if (LOG.isLoggable(Level.INFO)) {
String std = (ret == 0) ? Binaries.STDOUT : Binaries.STDERR;
FilenameFilter ff = new WildcardFileFilter("*" + std, IOCase.INSENSITIVE);
File[] f = outFile.getParentFile().listFiles(ff);
if (f != null && f.length > 0)
LOG.info(f[0] + ":\n" + FileUtils.readFileToString(f[0], "UTF-8"));
}
}
}
/**
* Get a file resources.
* @param id
* @param workspace
* @param LOG
* @param service
* @return
* @throws csip.api.server.ServiceException
*/
public static Executable getResourceExe0(Class<?> service, String id,
final File workspace, SessionLogger LOG) throws ServiceException {
Resource resource = getResourceById(service, id);
if (resource == null)
throw new ServiceException("Not found: " + id);
return getResourceExe0(resource, workspace, LOG, service);
}
public static Executable getResourceExe0(Resource resource,
final File workspace, SessionLogger LOG, Class<?> service) throws ServiceException {
File executable = null;
String file = resource.file();
switch (resource.type()) {
case REFERENCE:
if (resource.file() == null || resource.file().isEmpty())
return null;
file = Utils.resolve(file);
executable = new File(file);
// check only if there is a reference to an absolute file.
// relative commads could use the PATH.
if (executable.isAbsolute() && !executable.exists())
throw new ServiceException("Referenced executable not found: " + file);
break;
case EXECUTABLE:
try {
File csip_home = new File(Config.getString(CSIP_DIR));
if (file == null || file.isEmpty()) {
if (service == null)
return null;
file = '/' + service.getName().replace('.', '/') + ".exe";
File f = new File(csip_home, file);
try {
executable = unpackResource0(file, ResourceType.EXECUTABLE, csip_home, LOG);
} catch (IOException E) {
throw new ServiceException(E);
}
// try again....
if (!f.exists() || !f.canRead())
throw new ServiceException("Cannot find python file: '" + f + "'");
} else {
file = Utils.resolve(file);
executable = unpackResource0(file, ResourceType.EXECUTABLE, csip_home, LOG);
}
} catch (IOException ex) {
throw new ServiceException(ex);
}
break;
default:
throw new ServiceException("Illegal resource type: " + resource.type());
}
if (executable == null)
throw new ServiceException("executable is null:for " + file);
if (!executable.canExecute())
executable.setExecutable(true, true);
Execution p = null;
List<Object> args = new ArrayList<>();
if (resource.wine() && File.pathSeparatorChar == ':') {
p = new Execution(new File(Config.getString("wine.path", "/usr/bin/wine")));
args.add(executable.toString());
} else {
p = new Execution(new File(executable.toString()));
}
if (!resource.args().isEmpty())
args.add(resource.args().split("\\s+"));
if (args.size() > 0)
p.setArguments(args);
if (resource.env().length > 0)
p.environment().putAll(parseEnv(resource.env()));
return asExecutable(p, workspace, LOG);
}
/**
* Get merged Resources from cache.
* @param c
* @return
*/
public static List<Resource> getMergedResources(Class<?> c) {
return rc.get(c, Binaries::getMergedResources0);
}
/**
* Get merged Resources
* @param c
* @return
*/
private static List<Resource> getMergedResources0(Class<?> c) {
Resource[] r = c.getAnnotationsByType(Resource.class);
if (r == null || r.length == 0)
return NO_RES;
List<Resource> l = new ArrayList<>();
for (Resource res : r) {
if (res.from() != Object.class)
l.addAll(getMergedResources0(res.from()));
else
l.add(res);
}
return l;
}
/**
* Get Resource by type.
* @param c
* @param type
* @return
*/
public static List<Resource> getResourcesByType(Class<?> c, ResourceType type) {
List<Resource> mr = getMergedResources(c);
if (mr == NO_RES)
return mr;
List<Resource> l = new ArrayList<>();
for (Resource resource : mr) {
if (resource.type() == type)
l.add(resource);
}
return l;
}
/**
* Get the resource by ID.
*
* @param c
* @param id
* @return
*/
public static Resource getResourceById(Class<?> c, String id) {
for (Resource resource : getMergedResources(c)) {
if (id.equals(resource.id()))
return resource;
}
return null;
}
/**
* Get the auto execution resource.
*
* @param c Class e
* @return
*/
public static Resource getAutoExecResource(Class<?> c) {
for (Resource resource : getMergedResources(c)) {
if ((resource.id().equals("") || resource.id().equals(AUTO_RUN))
&& ((resource.type() == ResourceType.PYTHON3)
|| resource.type() == ResourceType.PYTHON2
|| resource.type() == ResourceType.EXECUTABLE
|| resource.type() == ResourceType.CLASSNAME
// || resource.type() == ResourceType.OMS_COMP
// || resource.type() == ResourceType.OMS_DSL)
))
return resource;
}
return null;
}
public static void extractAllResources(Class<?> c, SessionLogger LOG) throws ServiceException {
// do this only once per class, hence the SimpleCache
extractCache.get(c, (Class<?> k) -> {
File csip_home = new File(Config.getString(CSIP_DIR));
csip_home.mkdirs();
for (Resource resource : getMergedResources(c)) {
if (resource.type() == ResourceType.REFERENCE
|| resource.type() == ResourceType.JDBC
|| resource.type() == ResourceType.OUTPUT
|| resource.type() == ResourceType.INSTALL
|| resource.file().isEmpty())
continue;
try {
unpackResource0(resource, csip_home, LOG);
} catch (IOException ex) {
LOG.log(Level.SEVERE, null, ex);
}
}
return true;
});
}
public static File unpackResource0(String name, ResourceType resType,
File file_or_folder, SessionLogger LOG) throws IOException {
name = Utils.resolve(name.trim());
LOG.info("accessing resource : " + name);
File outFile = file_or_folder;
if (file_or_folder.isDirectory())
outFile = new File(file_or_folder, name);
synchronized (resource_lock) {
// check by date, ignore milliseconds since they might be not stored.
if (outFile.exists() && (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l)))
return outFile;
// extract the whole folder.and anything in it.
if (name.endsWith("/**") && (resType == ResourceType.ARCHIVE)) {
name = name.substring(0, name.length() - 3);
URL url = Binaries.class.getResource(name);
File file = null;
try {
file = new File(url.toURI());
} catch (URISyntaxException e) {
file = new File(url.getPath());
}
if (!(file.exists() && file.isDirectory()))
throw new IllegalArgumentException("No such directory: " + name);
FileUtils.copyToDirectory(file, file_or_folder);
return file_or_folder;
}
if (!outFile.getParentFile().exists())
outFile.getParentFile().mkdirs();
URL u = Binaries.class.getResource(name);
if (u == null)
throw new IllegalArgumentException("No such resource: " + name);
InputStream in = new BufferedInputStream(u.openStream());
FileUtils.copyInputStreamToFile(in, outFile);
in.close();
if (resType == ResourceType.EXECUTABLE) {
if (!outFile.canExecute())
outFile.setExecutable(true, true);
}
outFile.setReadable(true);
outFile.setLastModified(DATE_AT_STARTUP);
LOG.info("extracted: " + name + " to " + outFile);
// file was an archive, unzip it.
if (name.endsWith(".zip") && (resType == ResourceType.ARCHIVE)) {
ZipFiles.unzip(outFile);
LOG.info("unzipped: " + outFile);
}
return outFile;
}
}
/**
* This is needed because of services calling the above method.
*
* @param r resource
* @param LOG SessionLogger
* @param file_or_folder input directory/file
* @return unpacked resource
* @throws IOException required by unpackResource0
*/
public static File unpackResource0(Resource r, File file_or_folder, SessionLogger LOG) throws IOException {
return unpackResource0(r.file(), r.type(), file_or_folder, LOG);
}
/**
* get the FS usage of that directory/file location in percent.
*
* @param file input directory/file
* @return value 0.0 .. 1.0, 1.0 means 100% full.
*/
public static double getFSUsage(File file) {
return 1 - (double) file.getUsableSpace() / file.getTotalSpace();
}
/**
* get the library jars
*
* @param dir dir where to look for jar libraries
* @return the list of jars
*/
public static List<File> getJars(File dir) {
File[] f = dir.listFiles((FilenameFilter) new WildcardFileFilter("*.jar"));
if (f == null)
return new ArrayList<>();
return Arrays.asList(f);
}
/**
* Convert the files to a classpath
*
* @param f list of files
* @return classpath of jar files
*/
public static String asClassPath(List<File> f) {
StringBuilder b = new StringBuilder();
for (int i = 0; i < f.size(); i++) {
if (f.get(i).getName().endsWith(".jar")) {
b.append(f.get(i).toString());
if (i < f.size() - 1)
b.append(File.pathSeparatorChar);
}
}
return b.toString();
}
/**
* Converts a map to system properties.
*
* @param m input map
* @return system properties
*/
public static String[] asSysProps(Map<String, String> m) {
List<String> b = new ArrayList<>();
for (String key : m.keySet()) {
String val = m.get(key);
b.add("-D" + key + "=" + val);
}
return b.toArray(new String[b.size()]);
}
/**
* Parse environmental variables to be used for execution
*
* @param env environmental variables
* @return map of variable name and value
*/
public static Map<String, String> parseEnv(String[] env) {
Map<String, String> m = new HashMap<>();
if (env == null)
return m;
for (String s : env) {
if (s == null || s.isEmpty())
continue;
String t[] = s.trim().split("\\s*=\\s*");
if (t.length == 2)
m.put(t[0], Utils.resolve(t[1]));
}
return m;
}
/**
* Parses a string into a number of bytes. e.g. "1KB" = 1024,
* @param in input string
* @return the size in bytes.
*/
@SuppressWarnings("fallthrough")
public static long parseByteSize(String in) {
in = in.trim().replaceAll(",", ".");
try {
return Long.parseLong(in);
} catch (NumberFormatException e) {
}
Matcher m = Pattern.compile("([\\d.,]+)\\s*(\\w)").matcher(in);
m.find();
double scale = 1;
switch (m.group(2).charAt(0)) {
case 'G':
case 'g':
scale *= 1024;
case 'M':
case 'm':
scale *= 1024;
case 'K':
case 'k':
scale *= 1024;
break;
default:
throw new IllegalArgumentException(in);
}
return Math.round(Double.parseDouble(m.group(1)) * scale);
}
}