Binaries.java [src/csip/utils] Revision: default  Date:
/*
 * $Id$
 *
 * This file is part of the Cloud Services Integration Platform (CSIP),
 * a Model-as-a-Service framework, API and application suite.
 *
 * 2012-2022, Olaf David and others, OMSLab, Colorado State University.
 *
 * OMSLab licenses this file to you under the MIT license.
 * See the LICENSE file in the project root for more information.
 */
package csip.utils;

import csip.Config;
import static csip.Config.CSIP_DIR;
import static csip.Config.CSIP_JDBC_CHECKVALID;
import csip.api.server.Executable;
import csip.api.server.ServiceException;
import csip.SessionLogger;
import csip.Utils;
import csip.annotations.Resource;
import csip.annotations.ResourceType;
import static csip.annotations.ResourceType.CLASSNAME;
import static csip.annotations.ResourceType.JAR;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException;
import java.net.URL;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.sql.DataSource;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.tomcat.jdbc.pool.DataSourceProxy;
import org.apache.tomcat.jdbc.pool.PoolProperties;

/**
 *
 * @author od
 */
public class Binaries {

  public static final String STDOUT = "-stdout.txt";
  public static final String STDERR = "-stderr.txt";

  public static final String AUTO_RUN = "auto";

  public static final String DRIVER = "driverClassName";

  /**
   * JDBC Pool class.
   */
  private static class JDBCPool {

    DataSource datasource;
    String url;
    String resolved_url = "";
    Map<String, String> env;


    /**
     * Create a pool
     * @param url
     * @param env
     */
    JDBCPool(String url, Map<String, String> env) {
      this.url = url;
      this.env = env;
    }


    /**
     * Get the pool connection.
     *
     * @param logger
     * @return
     * @throws ServiceException
     */
    Connection getConnection(SessionLogger logger) throws ServiceException {
      try {
        // probe for configuration change.
        String res_url = Utils.resolve(url);
        if (datasource == null || !res_url.equals(resolved_url)) {
          //Close previous datasource if it exists
          if (datasource != null)
            shutdown();

          //Remember this connection string in case it has changed.
          resolved_url = res_url;
          PoolProperties p = new PoolProperties();
          defaultProperties(p);
          p.setUrl(resolved_url);
          logger.info(" JDBC pool property:  url:" + resolved_url);

          // load the driver if not already done do
          if (!env.containsKey(DRIVER)) {
            String rurl = resolved_url.toLowerCase();
            if (rurl.contains(":postgresql:"))
              env.put(DRIVER, "org.postgresql.Driver");
            else if (rurl.contains(":sqlserver:"))
              env.put(DRIVER, "com.microsoft.sqlserver.jdbc.SQLServerDriver");
            else if (rurl.contains(":mysql:"))
              env.put(DRIVER, "com.mysql.jdbc.Driver");
            else if (rurl.contains(":sqlite:"))
              env.put(DRIVER, "org.sqlite.JDBC");
            else if (rurl.contains(":oracle:"))
              env.put(DRIVER, "oracle.jdbc.driver.OracleDriver");
            else if (rurl.contains(":db2:"))
              env.put(DRIVER, "com.ibm.db2.jcc.DB2Driver");
            else if (rurl.contains(":sdm:"))
              env.put(DRIVER, "csip.sdm.SDMDriver");

          }

          // overwrite the defaults.
          Properties dbp = new Properties();
          for (String key : env.keySet()) {
            String val = env.get(key);
            try {
              BeanUtils.setProperty(p, key, val);
              logger.info("Set JDBC pool property: " + key + ": " + val);
            } catch (IllegalAccessException | InvocationTargetException E) {
              dbp.setProperty(key, val);
              logger.info("Set DB property: " + key + ": " + val);
            }
          }
          if (!dbp.isEmpty())
            p.setDbProperties(dbp);

          logger.info("JDBC Pool properties:" + p.toString());
          datasource = new org.apache.tomcat.jdbc.pool.DataSource(p);
          logger.info("Created JDBC datasource: " + datasource.toString());
        }
        return datasource.getConnection();
      } catch (SQLException ex) {
        logger.log(Level.SEVERE, null, ex);
        throw new ServiceException("Unable to connect..  Please check the configuration parameter");
      }
    }


    /**
     * set the default properties.
     *
     * @param p
     */
    private void defaultProperties(PoolProperties p) {
      p.setDefaultAutoCommit(false);
      p.setJmxEnabled(false);
      p.setTestOnBorrow(true);
      p.setValidationQuery("SELECT 1");
      p.setTestOnReturn(false);
      p.setValidationInterval(30000);
      p.setMaxWait(10000);
      p.setRemoveAbandonedTimeout(60);
      p.setRemoveAbandoned(true);
      p.setInitialSize(10);
      p.setMaxActive(250);
      p.setMaxIdle(100);
      p.setMinIdle(10);
      p.setSuspectTimeout(60);
      p.setTimeBetweenEvictionRunsMillis(30000);
      p.setMinEvictableIdleTimeMillis(60000);
    }


    /**
     * Shutdown this pool.
     */
    void shutdown() {
      if (datasource instanceof DataSourceProxy) {
        ((DataSourceProxy) datasource).close(true);
        System.out.println("Closed datasource " + datasource.toString());
      }
      datasource = null;
    }
  }

  /**
   * Map of JDBCPools.
   */
  private static final Map<String, JDBCPool> jdbcPools = new HashMap<>();
  private static final List<Resource> NO_RES = new ArrayList<>();

  private static final Object resource_lock = new Object();

  private static final long DATE_AT_STARTUP = new Date().getTime();
  private static final long DATE_AT_STARTUP_SEC = DATE_AT_STARTUP / 1000l;

  private static SimpleCache<Class<?>, List<Resource>> rc = new SimpleCache<>();
  private static SimpleCache<Class<?>, Boolean> extractCache = new SimpleCache<>();

  static boolean newCP = Config.getBoolean("csip.newcp", false);


  /**
   * Shuts down all JDBC pools.
   */
  public static void shutdownJDBC() {
    if (newCP) {
      ConnectionPools.singleton().shutdownJDBC();
      return;
    }
    synchronized (jdbcPools) {
      for (JDBCPool pool : jdbcPools.values()) {
        pool.shutdown();
      }
      jdbcPools.clear();
    }
  }


  /**
   * Get the architecture
   * @return the architecture
   */
  public static String getArch() {
    String os = System.getProperty("os.name").toLowerCase();
    if (os.contains("bsd"))
      os = "bsd";
    else
      os = os.substring(0, 3);
    return os + "-" + System.getProperty("os.arch").toLowerCase();
  }


  /**
   *
   * @param p1
   * @param workspace
   * @param LOG
   * @return
   */
  private static Executable asExecutable(final Execution p1, final File workspace, final SessionLogger LOG) {

    return new Executable() {
      File stdout = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDOUT);
      File stderr = new File(workspace, p1.getFile().getName() + "-" + p1.hashCode() + STDERR);

      Writer out;
      Writer err;


      @Override
      public File stdout() {
        return stdout;
      }


      @Override
      public File stderr() {
        return stderr;
      }


      @Override
      public String getName() {
        return p1.getFile().toString();
      }


      @Override
      public void setArguments(Object... args) {
        List<Object> l = new ArrayList<>();
        Object[] o = p1.getArguments();
        if (o.length > 0 && o[0] instanceof String && p1.getFile().getName().contains("wine"))
          // keep the first argument, if execution is wine.
          l.add(o[0]);
        l.addAll(Arrays.asList(args));
        p1.setArguments(l);
      }


      @Override
      public void addArguments(Object... args) {
        List<Object> l = new ArrayList<>();
        l.addAll(Arrays.asList(p1.getArguments()));
        l.addAll(Arrays.asList(args));
        p1.setArguments(l);
      }


      @Override
      public Object[] getArguments() {
        return p1.getArguments();
      }


      @Override
      public Map<String, String> environment() {
        return p1.environment();
      }


      @Override
      public int exec() throws IOException {
        p1.redirectOutput(out != null ? out : new FileWriter(stdout()));
        p1.redirectError(err != null ? err : new FileWriter(stderr()));
        p1.setWorkingDirectory(workspace);
        p1.setLogger(LOG);

        int ret = p1.exec();
        if (ret != 0 && LOG.isLoggable(Level.SEVERE) && stderr().exists()) {
          String s = FileUtils.readFileToString(stderr(), "UTF-8");
          LOG.severe(s);
        }
        return ret;
      }


      @Override
      public void redirectDefaults() throws IOException {
        out = new FileWriter(stdout(), true);
        err = new FileWriter(stderr(), true);
      }


      @Override
      public void redirectOutput(String file) throws IOException {
        if (file == null)
          throw new IOException("missing file name for redirect.");
        out = new FileWriter(new File(workspace, file));
      }


      @Override
      public void redirectError(String file) throws IOException {
        if (file == null)
          throw new IOException("missing file name for redirect.");
        err = new FileWriter(new File(workspace, file));
      }


      @Override
      public void redirectOutput(StringWriter w) throws IOException {
        out = w;
      }


      @Override
      public void redirectError(StringWriter w) throws IOException {
        err = w;
      }


      @Override
      public void setStdoutHandler(Executable.StdHandler handler) {
        p1.setStdoutHandler(handler);
      }


      @Override
      public void setStderrHandler(Executable.StdHandler handler) {
        p1.setStderrHandler(handler);
      }


      @Override
      public void setTimeout(long time, TimeUnit unit) {
        p1.setTimeout(time, unit);
      }

    };
  }


  /**
   *
   * @param c
   * @param id
   * @return
   * @throws ServiceException
   */
  public static Resource getResource(Class<?> c, String id) throws ServiceException {
    return getResourceById(c, id);
  }


  /**
   * Get a file resources.
   * @param c the service class
   * @param id the Resource ID
   * @return the file.
   * @throws csip.api.server.ServiceException
   */
  public static File getResourceFile(Class<?> c, String id) throws ServiceException {
    Resource resource = getResourceById(c, id);
    if (resource == null)
      throw new ServiceException("No such resource: '" + id + "'");

    String file = resource.file();
    if (file == null || file.isEmpty())
      return null;

    file = Utils.resolve(file);
    if (resource.type() == ResourceType.REFERENCE) {
      File f = new File(file);
      if (!f.exists() || !f.canRead())
        throw new ServiceException("Cannot find file: '" + f + "'");

      return f;
    }

    File csip_home = new File(Config.getString(CSIP_DIR));
    if (file.endsWith("zip") && resource.type() == ResourceType.ARCHIVE)
      return new File(csip_home, file.substring(0, file.indexOf('.')));

    File f = new File(csip_home, file);
    if (!f.exists() || !f.canRead())
      throw new ServiceException("Cannot find file: '" + f + "'");

    return f;
  }


  /**
   * Get the resource as java file.
   *
   * @param c the service class
   * @param id the resource id
   * @param workspace the session workspace
   * @param jars the jar files
   * @param LOG Session Logger
   * @return the Executable
   * @throws ServiceException
   */
  public static Executable getResourceJava(Class<?> c, String id, final File workspace,
      List<File> jars, SessionLogger LOG) throws ServiceException {
    Resource r = getResourceById(c, id);
    if (r == null || r.type() != ResourceType.CLASSNAME)
      throw new ServiceException("Not found: " + id);

    return getResourceJava(r, workspace, jars, LOG);
  }


  public static Executable getResourceJava(Resource r, final File workspace,
      List<File> jars, SessionLogger LOG) throws ServiceException {
    String oms_java_home = Config.getString("java.home", "/opt/jdk1.7.0_45");
    Execution p = new Execution(new File(oms_java_home, "/bin/java"));
    p.setArguments(//r[0].args().split("\\s+"), // no jvmoptions
        "-cp", asClassPath(jars),
        r.file(),
        r.args().split("\\s+")
    );
    if (r.env().length > 0)
      p.environment().putAll(parseEnv(r.env()));

    return asExecutable(p, workspace, LOG);
  }


  public static Executable getResourceJava0(Resource r, File workspace,
      SessionLogger LOG) throws ServiceException {
    if (r.type() != CLASSNAME && r.type() != JAR)
      throw new IllegalArgumentException("Invalid java resource type");

    String oms_java_home = Config.getString("java.home");
    Execution p = new Execution(new File(oms_java_home, "/bin/java"));
    p.setArguments(
        r.type() == JAR ? "-jar" : "",
        Utils.resolve(r.file()),
        r.args().split("\\s+")
    );
    if (r.env().length > 0)
      p.environment().putAll(parseEnv(r.env()));
    return asExecutable(p, workspace, LOG);
  }


  static String adjustPathSeparator(String s) {
    return s.replace(';', File.pathSeparatorChar).replace(':', File.pathSeparatorChar);
  }


  private static Executable getScriptResource(Resource r,
      SessionLogger LOG, Class<?> service, File workspace,
      String interpreter, String ext, ResourceType t) throws ServiceException {

    Execution p = new Execution(new File(interpreter));

    String file = r.file();
    if (file == null || file.isEmpty()) {
      if (service == null)
        return null;

      file = '/' + service.getName().replace('.', '/') + ext;
    }
    file = Utils.resolve(file);
    File f = new File(Config.getString(CSIP_DIR), file);
    File csip_home = new File(Config.getString(CSIP_DIR));
    try {
      unpackResource0(file, t, csip_home, LOG);
    } catch (IOException E) {
      throw new ServiceException(E);
    }
    // try again....
    if (!f.exists() || !f.canRead())
      throw new ServiceException("Cannot find file: '" + f + "'");

    p.setArguments(f.toString(), r.args().split("\\s+"));
    if (r.env().length > 0)
      p.environment().putAll(parseEnv(r.env()));

    return asExecutable(p, workspace, LOG);
  }


  public static Executable getResourcePython(Resource r, final File workspace,
      SessionLogger LOG, Class<?> service) throws ServiceException {
    return getScriptResource(r, LOG, service, workspace,
        r.type() == ResourceType.PYTHON3
        ? Config.getString("csip.python3.path", "python3")
        : Config.getString("csip.python.path", "python"), ".py", ResourceType.PYTHON3);
  }


  public static Executable getResourceRScript(Resource r, final File workspace,
      SessionLogger LOG, Class<?> service) throws ServiceException {
    return getScriptResource(r, LOG, service, workspace,
        Config.getString("csip.rscript.path", "Rscript"), ".r", ResourceType.RSCRIPT);
  }


  public static Executable getResourceBash(Resource r, final File workspace,
      SessionLogger LOG, Class<?> service) throws ServiceException {
    return getScriptResource(r, LOG, service, workspace,
        Config.getString("csip.bash.path", "bash"), ".sh", ResourceType.RSCRIPT);
  }


//  public static Executable getResourcePython0(Resource r, final File workspace,
//      SessionLogger LOG, Class<?> service) throws ServiceException {
//    Execution p = new Execution(new File(r.type() == ResourceType.PYTHON3
//        ? Config.getString("csip.python3.path", "python3")
//        : Config.getString("csip.python.path", "python")));
//
//    String file = r.file();
//    if (file == null || file.isEmpty()) {
//      if (service == null) {
//        return null;
//      }
//      file = '/' + service.getName().replace('.', '/') + ".py";
//    }
//    file = Utils.resolve(file);
//    File f = new File(Config.getString(CSIP_DIR), file);
//    File csip_home = new File(Config.getString(CSIP_DIR));
//    try {
//      unpackResource0(file, ResourceType.PYTHON3, csip_home, LOG);
//    } catch (IOException E) {
//      throw new ServiceException(E);
//    }
//    // try again....
//    if (!f.exists() || !f.canRead()) {
//      throw new ServiceException("Cannot find file: '" + f + "'");
//    }
//    p.setArguments(f.toString(), r.args().split("\\s+"));
//    if (r.env().length > 0) {
//      p.environment().putAll(parseEnv(r.env()));
//    }
//    return asExecutable(p, workspace, LOG);
//  }
//
//
//  public static Executable getResourceRScript(Resource r, final File workspace,
//      SessionLogger LOG, Class<?> service) throws ServiceException {
//    Execution p = new Execution(new File(Config.getString("csip.rscript.path", "rscript")));
//
//    String file = r.file();
//    if (file == null || file.isEmpty()) {
//      if (service == null) {
//        return null;
//      }
//      file = '/' + service.getName().replace('.', '/') + ".r";
//    }
//    file = Utils.resolve(file);
//    File f = new File(Config.getString(CSIP_DIR), file);
//    File csip_home = new File(Config.getString(CSIP_DIR));
//    try {
//      unpackResource0(file, ResourceType.RSCRIPT, csip_home, LOG);
//    } catch (IOException E) {
//      throw new ServiceException(E);
//    }
//    // try again....
//    if (!f.exists() || !f.canRead()) {
//      throw new ServiceException("Cannot find file: '" + f + "'");
//    }
//    p.setArguments(f.toString(), r.args().split("\\s+"));
//    if (r.env().length > 0) {
//      p.environment().putAll(parseEnv(r.env()));
//    }
//    return asExecutable(p, workspace, LOG);
//  }
//  /**
//   * Get the resource as DSL.
//   *
//   * @param c the service class
//   * @param id the resource id
//   * @param workspace the workspace
//   * @param jars the jar files
//   * @param LOG logger
//   * @return the Executable interface
//   * @throws ServiceException
//   */
//  public static Executable getResourceOMSDSL(Class<?> c, String id,
//      final File workspace, List<File> jars, SessionLogger LOG) throws ServiceException {
//    Resource r = getResourceById(c, id);
//    if (r == null || r.type() != ResourceType.OMS_DSL)
//      throw new ServiceException("Not found: " + id);
//
//    return getResourceOMSDSL(r, workspace, jars, LOG);
//  }
//
//
//  public static Executable getResourceOMSDSL(Resource r,
//      final File workspace, List<File> jars, SessionLogger LOG) throws ServiceException {
//    String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
//    Execution p = new Execution(new File(oms_java_home, "/bin/java"));
//    p.setArguments(r.args().split("\\s+"), // jvmoptions
//        "-cp", asClassPath(jars),
//        "oms3.CLI",
//        "-r", r.file()
//    );
//    if (r.env().length > 0)
//      p.environment().putAll(parseEnv(r.env()));
//
//    return asExecutable(p, workspace, LOG);
//  }
  /**
   *
   * @param r
   * @param options
   * @param workspace
   * @param jars
   * @param loglevel
   * @param LOG
   * @return
   * @throws ServiceException
   */
  public static Executable getResourceOMSDSL(File r, Object[] options,
      final File workspace, List<File> jars, String loglevel, SessionLogger LOG) throws ServiceException {
    String oms_java_home = Config.getString("oms.java.home", "/opt/jdk1.7.0_45");
    Execution p = new Execution(new File(oms_java_home, "/bin/java"));
    p.setArguments(options, // jvmoptions
        "-cp", asClassPath(jars),
        "oms3.CLI",
        "-l", loglevel,
        "-r", r.toString()
    );
    return asExecutable(p, workspace, LOG);
  }


  /**
   * Get the JDBC connection from a resource definition.
   *
   * @param c
   * @param id
   * @param LOG
   * @return
   * @throws ServiceException
   */
  public static Connection getResourceJDBC(Class<?> c, String id, SessionLogger LOG) throws ServiceException {
    if (newCP)
      return ConnectionPools.singleton().getResourceJDBC(c, id, LOG);

    synchronized (jdbcPools) {
      JDBCPool p = jdbcPools.get(id);
      if (p == null) {
        Resource r = getResourceById(c, id);
        if (r != null) {
          if (r.type() == ResourceType.JDBC) {
            String url = r.file();
            if (url != null && !url.isEmpty()) {
              Map<String, String> env = parseEnv(r.env());
              p = new JDBCPool(url, env);
              Connection con = p.getConnection(LOG);
              try {
                int valid = Config.getInt(CSIP_JDBC_CHECKVALID, -1); // default: no checking
                if (valid < -1)
                  valid = -1;

                if (valid == -1 || con.isValid(valid)) {
                  jdbcPools.put(id, p);
                  return con;
                } else {
                  throw new ServiceException("Cannot connect, invalid connection to: " + url);
                }
              } catch (SQLException ex) {
                throw new ServiceException("Invalid timeout: " + url);
              }
            } else {
              throw new ServiceException("No url connection string in 'file': " + id);
            }
          } else {
            throw new ServiceException("Not a JDBC resource: " + id);
          }
        } else {
          throw new ServiceException("No such resource: " + id);
        }
      }
      return p.getConnection(LOG);
    }
  }


  /**
   * Add to JDBC pool
   *
   * @param id
   * @param url
   */
  public static void addToJDBCPool(String id, String url) {
    if (newCP)
      ConnectionPools.singleton().addToJDBCPool(id, url);
    else
      jdbcPools.put(id, new JDBCPool(url, new HashMap<>()));
  }


  /**
   * Add to JDBC pool
   *
   * @param id
   * @param url
   * @param env
   */
  public static void addToJDBCPool(String id, String url, Map<String, String> env) {
    if (newCP)
      ConnectionPools.singleton().addToJDBCPool(id, url, env);
    else
      jdbcPools.put(id, new JDBCPool(url, env));
  }


  /**
   * Get Connection.
   *
   * @param id
   * @param logger
   * @return
   * @throws ServiceException
   */
  public static Connection getConnection(String id, SessionLogger logger) throws ServiceException {
    if (newCP)
      return ConnectionPools.singleton().getConnection(id, logger);

    JDBCPool p = jdbcPools.get(id);
    if (p == null)
      throw new ServiceException("No such resource: " + id);

    return p.getConnection(logger);
  }


  public static void doInstall(Resource resource, SessionLogger LOG) throws IOException {
    String file = resource.file();
    if (file.isEmpty())
      return;

    file = Utils.resolve(file);
    File csip_home = new File(Config.getString(CSIP_DIR));
    File outFile = new File(csip_home, file);

    synchronized (resource_lock) {
      // check by date, ignore milliseconds since they might be not stored.
      if (outFile.exists() && (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l)))
        return;

      File executable = unpackResource0(resource, csip_home, LOG);
      if (!executable.canExecute())
        executable.setExecutable(true, true);

      Execution p = null;
      List<Object> args = new ArrayList<>();
      if (resource.wine() && File.pathSeparatorChar == ':') {
        p = new Execution(new File(Config.getString("wine.path", "/usr/bin/wine")));
        args.add(executable.toString());
      } else {
        p = new Execution(new File(executable.toString()));
      }
      if (!resource.args().isEmpty())
        args.add(resource.args().split("\\s+"));

      if (args.size() > 0)
        p.setArguments(args);

      if (resource.env().length > 0)
        p.environment().putAll(parseEnv(resource.env()));

      LOG.info("executing install : " + outFile);
      Executable ex = asExecutable(p, outFile.getParentFile(), LOG);
      int ret = ex.exec();

      if (LOG.isLoggable(Level.INFO)) {
        String std = (ret == 0) ? Binaries.STDOUT : Binaries.STDERR;
        FilenameFilter ff = new WildcardFileFilter("*" + std, IOCase.INSENSITIVE);
        File[] f = outFile.getParentFile().listFiles(ff);
        if (f != null && f.length > 0)
          LOG.info(f[0] + ":\n" + FileUtils.readFileToString(f[0], "UTF-8"));
      }
    }
  }


  /**
   * Get a file resources.
   * @param id
   * @param workspace
   * @param LOG
   * @param service
   * @return
   * @throws csip.api.server.ServiceException
   */
  public static Executable getResourceExe0(Class<?> service, String id,
      final File workspace, SessionLogger LOG) throws ServiceException {
    Resource resource = getResourceById(service, id);
    if (resource == null)
      throw new ServiceException("Not found: " + id);
    return getResourceExe0(resource, workspace, LOG, service);
  }


  public static Executable getResourceExe0(Resource resource,
      final File workspace, SessionLogger LOG, Class<?> service) throws ServiceException {
    File executable = null;
    String file = resource.file();
    switch (resource.type()) {
      case REFERENCE:
        if (resource.file() == null || resource.file().isEmpty())
          return null;
        file = Utils.resolve(file);
        executable = new File(file);

        // check only if there is a reference to an absolute file.
        // relative commads could use the PATH.
        if (executable.isAbsolute() && !executable.exists())
          throw new ServiceException("Referenced executable not found: " + file);

        break;
      case EXECUTABLE:
        try {
          File csip_home = new File(Config.getString(CSIP_DIR));
          if (file == null || file.isEmpty()) {
            if (service == null)
              return null;

            file = '/' + service.getName().replace('.', '/') + ".exe";
            File f = new File(csip_home, file);
            try {
              executable = unpackResource0(file, ResourceType.EXECUTABLE, csip_home, LOG);
            } catch (IOException E) {
              throw new ServiceException(E);
            }
            // try again....
            if (!f.exists() || !f.canRead())
              throw new ServiceException("Cannot find python file: '" + f + "'");

          } else {
            file = Utils.resolve(file);
            executable = unpackResource0(file, ResourceType.EXECUTABLE, csip_home, LOG);
          }
        } catch (IOException ex) {
          throw new ServiceException(ex);
        }
        break;
      default:
        throw new ServiceException("Illegal resource type: " + resource.type());
    }
    if (executable == null)
      throw new ServiceException("executable is null:for " + file);
    if (!executable.canExecute())
      executable.setExecutable(true, true);

    Execution p = null;
    List<Object> args = new ArrayList<>();
    if (resource.wine() && File.pathSeparatorChar == ':') {
      p = new Execution(new File(Config.getString("wine.path", "/usr/bin/wine")));
      args.add(executable.toString());
    } else {
      p = new Execution(new File(executable.toString()));
    }
    if (!resource.args().isEmpty())
      args.add(resource.args().split("\\s+"));
    if (args.size() > 0)
      p.setArguments(args);
    if (resource.env().length > 0)
      p.environment().putAll(parseEnv(resource.env()));
    return asExecutable(p, workspace, LOG);
  }


  /**
   * Get merged Resources from cache.
   * @param c
   * @return
   */
  public static List<Resource> getMergedResources(Class<?> c) {
    return rc.get(c, Binaries::getMergedResources0);
  }


  /**
   * Get merged Resources
   * @param c
   * @return
   */
  private static List<Resource> getMergedResources0(Class<?> c) {
    Resource[] r = c.getAnnotationsByType(Resource.class);
    if (r == null || r.length == 0)
      return NO_RES;
    List<Resource> l = new ArrayList<>();
    for (Resource res : r) {
      if (res.from() != Object.class)
        l.addAll(getMergedResources0(res.from()));
      else
        l.add(res);
    }
    return l;
  }


  /**
   * Get Resource by type.
   * @param c
   * @param type
   * @return
   */
  public static List<Resource> getResourcesByType(Class<?> c, ResourceType type) {
    List<Resource> mr = getMergedResources(c);
    if (mr == NO_RES)
      return mr;
    List<Resource> l = new ArrayList<>();
    for (Resource resource : mr) {
      if (resource.type() == type)
        l.add(resource);
    }
    return l;
  }


  /**
   * Get the resource by ID.
   *
   * @param c
   * @param id
   * @return
   */
  public static Resource getResourceById(Class<?> c, String id) {
    for (Resource resource : getMergedResources(c)) {
      if (id.equals(resource.id()))
        return resource;
    }
    return null;
  }


  /**
   * Get the auto execution resource.
   *
   * @param c Class e
   * @return
   */
  public static Resource getAutoExecResource(Class<?> c) {
    for (Resource resource : getMergedResources(c)) {
      if ((resource.id().equals("") || resource.id().equals(AUTO_RUN))
          && ((resource.type() == ResourceType.PYTHON3)
          || resource.type() == ResourceType.PYTHON2
          || resource.type() == ResourceType.EXECUTABLE
          || resource.type() == ResourceType.CLASSNAME 
//          || resource.type() == ResourceType.OMS_COMP
          //          || resource.type() == ResourceType.OMS_DSL)
          ))
        
        
        return resource;
    }
    return null;
  }


  public static void extractAllResources(Class<?> c, SessionLogger LOG) throws ServiceException {
    // do this only once per class, hence the SimpleCache
    extractCache.get(c, (Class<?> k) -> {
      File csip_home = new File(Config.getString(CSIP_DIR));
      csip_home.mkdirs();
      for (Resource resource : getMergedResources(c)) {
        if (resource.type() == ResourceType.REFERENCE
            || resource.type() == ResourceType.JDBC
            || resource.type() == ResourceType.OUTPUT
            || resource.type() == ResourceType.INSTALL
            || resource.file().isEmpty())
          continue;
        try {
          unpackResource0(resource, csip_home, LOG);
        } catch (IOException ex) {
          LOG.log(Level.SEVERE, null, ex);
        }
      }
      return true;
    });
  }


  public static File unpackResource0(String name, ResourceType resType,
      File file_or_folder, SessionLogger LOG) throws IOException {

    name = Utils.resolve(name.trim());
    LOG.info("accessing resource : " + name);

    File outFile = file_or_folder;
    if (file_or_folder.isDirectory())
      outFile = new File(file_or_folder, name);

    synchronized (resource_lock) {
      // check by date, ignore milliseconds since they might be not stored.
      if (outFile.exists() && (DATE_AT_STARTUP_SEC == (outFile.lastModified() / 1000l)))
        return outFile;

      // extract the whole folder.and anything in it.
      if (name.endsWith("/**") && (resType == ResourceType.ARCHIVE)) {
        name = name.substring(0, name.length() - 3);
        URL url = Binaries.class.getResource(name);
        File file = null;
        try {
          file = new File(url.toURI());
        } catch (URISyntaxException e) {
          file = new File(url.getPath());
        }

        if (!(file.exists() && file.isDirectory()))
          throw new IllegalArgumentException("No such directory: " + name);

        FileUtils.copyToDirectory(file, file_or_folder);
        return file_or_folder;
      }

      if (!outFile.getParentFile().exists())
        outFile.getParentFile().mkdirs();

      URL u = Binaries.class.getResource(name);
      if (u == null)
        throw new IllegalArgumentException("No such resource: " + name);

      InputStream in = new BufferedInputStream(u.openStream());
      FileUtils.copyInputStreamToFile(in, outFile);
      in.close();

      if (resType == ResourceType.EXECUTABLE) {
        if (!outFile.canExecute())
          outFile.setExecutable(true, true);
      }
      outFile.setReadable(true);
      outFile.setLastModified(DATE_AT_STARTUP);
      LOG.info("extracted: " + name + " to " + outFile);

      // file was an archive, unzip it.
      if (name.endsWith(".zip") && (resType == ResourceType.ARCHIVE)) {
        ZipFiles.unzip(outFile);
        LOG.info("unzipped: " + outFile);
      }
      return outFile;
    }
  }


  /**
   * This is needed because of services calling the above method.
   *
   * @param r resource
   * @param LOG SessionLogger
   * @param file_or_folder input directory/file
   * @return unpacked resource
   * @throws IOException required by unpackResource0
   */
  public static File unpackResource0(Resource r, File file_or_folder, SessionLogger LOG) throws IOException {
    return unpackResource0(r.file(), r.type(), file_or_folder, LOG);
  }


  /**
   * get the FS usage of that directory/file location in percent.
   *
   * @param file input directory/file
   * @return value 0.0 .. 1.0, 1.0 means 100% full.
   */
  public static double getFSUsage(File file) {
    return 1 - (double) file.getUsableSpace() / file.getTotalSpace();
  }


  /**
   * get the library jars
   *
   * @param dir dir where to look for jar libraries
   * @return the list of jars
   */
  public static List<File> getJars(File dir) {
    File[] f = dir.listFiles((FilenameFilter) new WildcardFileFilter("*.jar"));
    if (f == null)
      return new ArrayList<>();

    return Arrays.asList(f);
  }


  /**
   * Convert the files to a classpath
   *
   * @param f list of files
   * @return classpath of jar files
   */
  public static String asClassPath(List<File> f) {
    StringBuilder b = new StringBuilder();
    for (int i = 0; i < f.size(); i++) {
      if (f.get(i).getName().endsWith(".jar")) {
        b.append(f.get(i).toString());
        if (i < f.size() - 1)
          b.append(File.pathSeparatorChar);

      }
    }
    return b.toString();
  }


  /**
   * Converts a map to system properties.
   *
   * @param m input map
   * @return system properties
   */
  public static String[] asSysProps(Map<String, String> m) {
    List<String> b = new ArrayList<>();
    for (String key : m.keySet()) {
      String val = m.get(key);
      b.add("-D" + key + "=" + val);
    }
    return b.toArray(new String[b.size()]);
  }


  /**
   * Parse environmental variables to be used for execution
   *
   * @param env environmental variables
   * @return map of variable name and value
   */
  public static Map<String, String> parseEnv(String[] env) {
    Map<String, String> m = new HashMap<>();
    if (env == null)
      return m;

    for (String s : env) {
      if (s == null || s.isEmpty())
        continue;

      String t[] = s.trim().split("\\s*=\\s*");
      if (t.length == 2)
        m.put(t[0], Utils.resolve(t[1]));

    }
    return m;
  }


  /**
   * Parses a string into a number of bytes. e.g. "1KB" = 1024,
   * @param in input string
   * @return the size in bytes.
   */
  @SuppressWarnings("fallthrough")
  public static long parseByteSize(String in) {
    in = in.trim().replaceAll(",", ".");
    try {
      return Long.parseLong(in);
    } catch (NumberFormatException e) {
    }
    Matcher m = Pattern.compile("([\\d.,]+)\\s*(\\w)").matcher(in);
    m.find();
    double scale = 1;
    switch (m.group(2).charAt(0)) {
      case 'G':
      case 'g':
        scale *= 1024;
      case 'M':
      case 'm':
        scale *= 1024;
      case 'K':
      case 'k':
        scale *= 1024;
        break;
      default:
        throw new IllegalArgumentException(in);
    }
    return Math.round(Double.parseDouble(m.group(1)) * scale);
  }
}