Config.java [src/csip] Revision: Date:
/*
* $Id$
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip;
import csip.api.client.ModelDataServiceCall;
import csip.ModelDataService.Task;
import csip.utils.Binaries;
import csip.utils.SimpleCache;
import java.io.File;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.servlet.ServletContext;
/**
* Global properties, adjustable at runtime.
*
* @author od
*/
public class Config {
private static final Properties p = new Properties();
private static final Properties allProps = new Properties();
private static final List<Task> tasks = Collections.synchronizedList(new ArrayList<Task>());
//
static private SimpleCache<String, AutoCloseable> backends = new SimpleCache<>();
//
private static ExecutorService exec;
private static Timer timer;
private static final Registry reg = new Registry();
static final Logger LOG = Logger.getLogger("csip-core");
private static final Object timerLock = new Object();
/**
* CSIP API version. Set by the system. <br>
*/
public static final String CSIP_PLATFORM_VERSION = "csip.platform.version";
public static final String CSIP_CONTEXT_VERSION = "csip.context.version";
/**
* CSIP platform Architecture. Set by the system.<br>
* e.g.: lin-amd46, win-x86,...
*/
public static final String CSIP_ARCH = "csip.arch";
/**
* Remote Access ACL. list of IPs or subnets that are allows to connect via
* UI. This ACL is not for the services, just service management. <br>
* default: "127.0.0.1/32" (localhost only)
*/
public static final String CSIP_REMOTE_ACL = "csip.remote.acl";
public static final String CSIP_TIMEZONE = "csip.timezone";
public static final String CSIP_LOGGING_LEVEL = "csip.logging.level";
public static final String CSIP_RESPONSE_STACKTRACE = "csip.response.stacktrace";
public static final String CSIP_RESPONSE_SQLFILTER = "csip.response.sqlfilter";
public static final String CSIP_RESPONSE_JSONINDENT = "csip.response.jsonindent";
public static final String CSIP_RESPONSE_ERROR_HTTPSTATUS = "csip.response.error.httpstatus";
public static final String CSIP_PAYLOAD_VERSION = "csip.payload.version";
//
public static final String CSIP_SESSION_BACKEND = "csip.session.backend";
public static final String CSIP_SESSION_TTL = "csip.session.ttl";
public static final String CSIP_SESSION_MONGODB_URI = "csip.session.mongodb.uri";
public static final String CSIP_SESSION_TTL_FAILED = "csip.session.ttl.failed";
public static final String CSIP_SESSION_TTL_CANCELLED = "csip.session.ttl.cancelled";
public static final String CSIP_ARCHIVE_BACKEND = "csip.archive.backend";
public static final String CSIP_ARCHIVE_MONGODB_URI = "csip.archive.mongodb.uri";
public static final String CSIP_ARCHIVE_MAX_FILE_SIZE = "csip.archive.max.filesize";
public static final String CSIP_ARCHIVE_TTL = "csip.archive.ttl";
public static final String CSIP_ARCHIVE_FAILEDONLY = "csip.archive.failedonly";
public static final String CSIP_ARCHIVE_ONREQUEST = "csip.archive.onrequest";
public static final String CSIP_RESULTSTORE_BACKEND = "csip.resultstore.backend";
public static final String CSIP_RESULTSTORE_MONGODB_URI = "csip.resultstore.mongodb.uri";
public static final String CSIP_RESULTSTORE_LIMIT = "csip.resultstore.limit";
//
public static final String CSIP_DIR = "csip.dir";
public static final String CSIP_BIN_DIR = "csip.bin.dir";
public static final String CSIP_WORK_DIR = "csip.work.dir";
public static final String CSIP_RESULTS_DIR = "csip.results.dir";
public static final String CSIP_CACHE_DIR = "csip.cache.dir";
public static final String CSIP_DATA_DIR = "csip.data.dir";
//
public static final String CSIP_SNAPSHOT = "csip.snapshot";
public static final String CSIP_KEEPWORKSPACE = "csip.keepworkspace";
public static final String CSIP_JDBC_CHECKVALID = "csip.jdbc.checkvalid";
// values for session/archive/result
public static final String MONGODB = "mongodb";
public static final String LOCAL = "local";
public static final String SQL = "sql";
public static final String NONE = "none";
public static final String KAFKA = "kafka";
// publisher
public static final String CSIP_PUBLISHER_BACKEND = "csip.publisher.backend";
public static final String CSIP_PUBLISHER_KAFKA_BOOTSTRAP_SERVERS = "csip.publisher.kafka.bootstrap_servers";
public static final String CSIP_PUBLISHER_KAFKA_ACKS = "csip.publisher.kafka.acks";
public static final String CSIP_PUBLISHER_KAFKA_TOPIC = "csip.publisher.kafka.topic";
public static final String CSIP_PUBLISHER_KAFKA_RETRIES = "csip.publisher.kafka.retries.";
public static final String CSIP_PUBLISHER_KAFKA_MAX_BLOCK_MS = "csip.publisher.kafka.max_block_ms.";
// Auth
public static final String CSIP_TOKEN_AUTHENTICATION = "csip.token.authentication";
// internal services
static final String CSIP_PUBSUB_ENABLED = "csip.pubsub.enabled";
static final String CSIP_DYNPY_ENABLED = "csip.dynpy.enabled";
public static Map<Object, Object> properties() {
return Collections.unmodifiableMap(p);
}
//
static SimpleCache<File, ReentrantLock> wsFileLocks = new SimpleCache<>();
public static Logger getSystemLogger(ModelDataServiceCall.ConfAccess a) {
if (a == null)
throw new NullPointerException("Illegal Access.");
return LOG;
}
static {
try {
/*
The CSIP version for platform and context (placeholder)
*/
put(CSIP_PLATFORM_VERSION, "$version: 2.6.28 default 808 83e2345ece11 2022-01-06 od, built at 2022-01-11 18:13 by od$");
put(CSIP_CONTEXT_VERSION, "$version: 0.0.0 000000000000");
put("csip.response.version", "true");
// put("csip.internal.uri", "https://129.82.10.125");
/*
* The runtime architecture.
*/
put(CSIP_ARCH, Binaries.getArch());
// for legacy settings only.
put("arch", Binaries.getArch());
// remote access acl
/*
remote access for UI and config. Provide a list of IPs or
subnets that are allows to connect. This ACL is not for the
services, just service management. The default is localhost only.
Example: "csip.remoteaccess.acl": "127.0.0.1/32 10.2.222.0/24"
*/
put(CSIP_REMOTE_ACL, "127.0.0.1 0:0:0:0:0:0:0:1");
// session
/*
The backend to use for session management. valid choices are
"mongodb", "sql", "local".
*/
put(CSIP_SESSION_BACKEND, LOCAL);
/*
The mongodb connecion string.
*/
put(CSIP_SESSION_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
The default time in seconds for a session to stay active after the
model finishes. All model results will be available for that period.
After this period expires the session will be removed and the model results
will be removed or archived. This value can be altered using
the "keep_results" metainfo value of a request.
see duration string examples: https://docs.oracle.com/javase/8/docs/api/java/time/Duration.html#parse-java.lang.CharSequence-
*/
put(CSIP_SESSION_TTL, "PT30S"); // 30 sec ttl
put(CSIP_SESSION_TTL_FAILED, "PT30S");
put(CSIP_SESSION_TTL_CANCELLED, "PT30S");
/*
The default csip timezone to be used for time management.
*/
put(CSIP_TIMEZONE, "MST7MDT");
/*
* Use http status code for the response, default is 200.
* If changed, this value should be > 500, otherwise it defaults
* to 200.
* If set to default and a service results in an error,
* the POST response will return a 200 status and the payload with the
* error message.
*
* If set to a value, e.g. "530" and a service results in an error,
* the POST response will return a 530 status and the payload with the
* error message. In addition, "400" will be returned if input
* is malformed or empty. Choose your code wisely!
*/
put(CSIP_RESPONSE_ERROR_HTTPSTATUS, "200");
/*
* Vary the payload sructure, JSONArray vs JSONObject for
* parameter/results entries. Defaults to version 1 (Array).
* JSONArray: "1"
* JSONObject: "2"
*/
put(CSIP_PAYLOAD_VERSION, "1");
// archive
/*
defines the archive backend implementation: "mongodb" or "none" are
possible.
"none" means disabled.
*/
put(CSIP_ARCHIVE_BACKEND, NONE);
/*
The mongodb connection uri, if the backend is set to "mongodb"
*/
put(CSIP_ARCHIVE_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
The max file size for an attachment to be archived.
*/
put(CSIP_ARCHIVE_MAX_FILE_SIZE, "10MB");
/*
The default time in seconds for an entry to stay in the archive.
All archived model results will be available for that period after
the session expired. After this period expires the archive will be removed.
see duration string examples: https://docs.oracle.com/javase/8/docs/api/java/time/Duration.html#parse-java.lang.CharSequence-.
*/
put(CSIP_ARCHIVE_TTL, "P1D"); // one day.
/*
If the archive is enabled, only archive failed runs, default: false
*/
put(CSIP_ARCHIVE_FAILEDONLY, "false");
/* set to true if archiving should be done only if requested
*/
put(CSIP_ARCHIVE_ONREQUEST, "false");
///// Publish
/* NONE or KAFKA
*/
put(CSIP_PUBLISHER_BACKEND, NONE);
// Authentication
/**
* NONE
*/
put(CSIP_TOKEN_AUTHENTICATION, NONE);
//// Logging
/*
The log level for service and system logging.
. All java.util.logging.Level log levels are usable.
*/
put(CSIP_LOGGING_LEVEL, "CONFIG");
/*
control if the stack trace should be part of the response metadata
if the logic fails. default is false. Enable this for debugging.
*/
put(CSIP_RESPONSE_STACKTRACE, "false");
/*
control if SQL error messages should be masked in the response payload.
default is true.
*/
put(CSIP_RESPONSE_SQLFILTER, "true");
/*
indentation of the response json, default is 2.
set it to 0 to minify the response json.
*/
put(CSIP_RESPONSE_JSONINDENT, "2");
/* Result store handling
*/
put(CSIP_RESULTSTORE_BACKEND, NONE);
put(CSIP_RESULTSTORE_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
The csip root directory
*/
put(CSIP_DIR, "/tmp/csip");
/*
The csip directories for executables.
*/
put(CSIP_BIN_DIR, "${csip.dir}/bin");
/*
The csip directory for sessions.
*/
put(CSIP_WORK_DIR, "${csip.dir}/work");
/*
The csip directory to store results.
*/
put(CSIP_RESULTS_DIR, "${csip.dir}/results");
/*
The csip cache file directory.
*/
put(CSIP_CACHE_DIR, "${csip.dir}/cache");
/*
The csip data directory.
*/
put(CSIP_DATA_DIR, "${csip.dir}/data");
// Core services as needed. need to be enabled in context config.s.
put(CSIP_DYNPY_ENABLED, "false");
put(CSIP_PUBSUB_ENABLED, "false");
/*
External url pats These
properties can be set to force a public scheme/host/protocol for
result file downloads and catalog listing. These properties
are not set per default. They can be set independently from
each other to change only selective parts of the URL.
If none of the propeties below are set the incomming URL is
being used to construct downloads and catalogs.
*/
// put("csip.public.scheme", ""); // e.g. https
// put("csip.public.host", ""); // e.g. csip.org
// put("csip.public.port", ""); // e.g. 8080 (-1 will remove the port in the url)
///////////////////////////////////////////////////////
//// more auxiliary properties
// thread management
put("codebase.threadpool", "32"); // 10 concurrent model runs.
put("codebase.url", "http://localhost:8080");
put("codebase.port", "8085");
put("codebase.localport", "8081");
put("codebase.servicename", "csip-vmscaler");
// wine
put("wine.path", "/usr/bin/wine");
//rusle2/weps
// put("r2.path", "/od/projects/csip.services/bin/RomeShell.exe"); // the path is the parent directory.
// put("r2.db", "http://oms-db.engr.colostate.edu/r2");
// put("weps.db", "http://oms-db.engr.colostate.edu/weps");
//oms related props
// put("oms.java.home", "/usr");
put("oms.java.home", "/opt/jdk1.8.0_51");
put("oms.esp.threads", "4");
// internal
put("vm.port", "8080");
update();
} catch (Exception E) {
Logger.getLogger(Config.class.getName()).log(Level.SEVERE, "Static init error", E);
}
}
private static String getVersion(String version) {
if (version == null || version.isEmpty())
return "?";
String v[] = version.split("\\s+");
if (v.length < 3)
return "?";
return v[1] + "(" + v[2] + ")";
}
private static String full_version;
static synchronized String getFullVersion() {
if (full_version == null)
full_version = getVersion(getString(CSIP_CONTEXT_VERSION)) + "-"
+ getVersion(getString(CSIP_PLATFORM_VERSION));
return full_version;
}
/**
* @param services the services to register
* @deprecated replace with Conf.register(resources, context) to filter and
* register.
*
*/
@Deprecated
public static void register(Set<Class<?>> services) {
getRegistry().register(services);
}
public static void register(Set<Class<?>> services, ServletContext context) {
ContextConfig.filterServices(context, services);
getRegistry().register(services);
}
/**
* @return The global registry.
* @deprecated use register() instead.
*/
@Deprecated
public static Registry registry() {
return reg;
}
static Registry getRegistry() {
return reg;
}
static boolean isArchiveEnabled() {
return !getString(CSIP_ARCHIVE_BACKEND, NONE).equals(NONE);
}
static boolean isResultStoreEnabled() {
return !getString(CSIP_RESULTSTORE_BACKEND, NONE).equals(NONE);
}
static synchronized SessionStore getSessionStore() {
return (SessionStore) backends.get(CSIP_SESSION_BACKEND, key -> {
String uri = null;
switch (getString(key)) {
case MONGODB:
uri = getString(CSIP_SESSION_MONGODB_URI);
if (uri == null)
throw new RuntimeException("missing uri configuration entry 'csip.session.mongodb.uri'");
return new MongoSessionStore(uri);
case SQL:
uri = getString("csip.session.sql.uri");
if (uri == null)
throw new RuntimeException("missing uri configuration entry 'csip.session.sql.uri'");
return new SQLSessionStore(uri);
case LOCAL:
return new LocalSessionStore();
default:
throw new RuntimeException("unknown session backend: " + getString(key));
}
});
}
static synchronized ArchiveStore getArchiveStore() {
return (ArchiveStore) backends.get(CSIP_ARCHIVE_BACKEND, key -> {
switch (getString(key)) {
case MONGODB:
String uri = getString(CSIP_ARCHIVE_MONGODB_URI);
if (uri == null)
throw new RuntimeException("missing uri configuration entry 'csip.archive.mongodb.uri'");
return new MongoArchiveStore1(uri);
case NONE:
return ArchiveStore.NONE;
default:
throw new RuntimeException("unknown archive backend: " + getString(key));
}
});
}
static synchronized Publisher getPublisher() {
return (Publisher) backends.get(CSIP_PUBLISHER_BACKEND, key -> {
switch (getString(key)) {
case KAFKA:
try {
return new KafkaPublisher(LOG, getString(CSIP_PUBLISHER_KAFKA_TOPIC),
getString(CSIP_PUBLISHER_KAFKA_BOOTSTRAP_SERVERS),
getString(CSIP_PUBLISHER_KAFKA_ACKS, "1"),
getInt(CSIP_PUBLISHER_KAFKA_RETRIES, 2),
getInt(CSIP_PUBLISHER_KAFKA_MAX_BLOCK_MS, 1000)
);
} catch (Exception E) {
LOG.log(Level.SEVERE, "Disabling kafka because of: ", E);
}
case NONE:
return Publisher.NONE;
default:
throw new RuntimeException("unknown publisher backend: " + getString(key));
}
});
}
static synchronized ResultStore getResultStore() {
return (ResultStore) backends.get(CSIP_RESULTSTORE_BACKEND, key -> {
switch (getString(key)) {
case MONGODB:
return new MongoResultStore(getString(CSIP_RESULTSTORE_MONGODB_URI));
case LOCAL:
return new MemResultStore(getInt(CSIP_RESULTSTORE_LIMIT, 64));
case NONE:
return ResultStore.NONE;
default:
throw new RuntimeException("unknown resultstore backend: " + getString(key));
}
});
}
static synchronized TokenAuthentication getTokenAuthentication() {
return (TokenAuthentication) backends.get(CSIP_TOKEN_AUTHENTICATION, key -> {
switch (getString(key)) {
case "property":
return new PropertyTokenAuthentication(Config.getString("csip.auth.tokens"));
case "jwt":
return new JWTAuthentication(Config.getString("csip.jwk.provider.url"));
case NONE:
return TokenAuthentication.NONE;
default:
throw new RuntimeException("Unknown Authentication backend: " + getString(CSIP_TOKEN_AUTHENTICATION));
}
});
}
static Timer getTimer() {
synchronized (timerLock) {
if (timer == null)
timer = new Timer();
return timer;
}
}
/**
* This is being called if schedule results in an illegal state exception.
*
* @return
*/
static Timer getNewTimer() {
synchronized (timerLock) {
LOG.info("Starting new timer for Session TTL handling.");
timer = new Timer();
return timer;
}
}
static synchronized ExecutorService getExecutorService() {
if (exec == null)
exec = Executors.newCachedThreadPool();
return exec;
}
static List<ModelDataService.Task> getModelTasks() {
return tasks;
}
static private void setupLogging() {
Level l = Level.parse(getString(CSIP_LOGGING_LEVEL));
LOG.log(Level.INFO, " LogLevel : {0}", l.toString());
Logger rootLogger = Logger.getLogger("");
rootLogger.setLevel(l);
for (Handler h : rootLogger.getHandlers()) {
h.setLevel(l);
}
LOG.setLevel(l);
}
/**
* Start up the servlet.
*
* @param context
*/
static void startup() {
setupLogging();
// getSessionStore().registerResources(true);
}
/**
* Shut down the servlet.
*
* @param context
*/
static void shutdown() {
tasks.forEach(task -> task.cancel());
reg.unregister();
if (exec != null) {
LOG.info("Shutting down ExecutorService");
exec.shutdownNow();
}
backends.forEach((name, feature) -> {
try {
LOG.info("Shutting backend '" + name + "'");
feature.close();
} catch (Exception ex) {
LOG.log(Level.SEVERE, null, ex);
}
});
// session.registerResources(false);
if (timer != null)
timer.cancel();
Binaries.shutdownJDBC();
}
/*
This is being called upon configuration update.
*/
static void update() {
rehashProperties();
}
public static boolean hasProperty(String key) {
return allProps.containsKey(key);
}
public static boolean isString(String key, String str) {
String s = getString(key);
return (s != null) && s.equals(str);
}
public static String getString(String key, String def) {
return getP(key, def);
}
public static String getString(String key) {
return getP(key, null);
}
public static boolean getBoolean(String key, boolean def) {
return Boolean.parseBoolean(getP(key, Boolean.toString(def)));
}
public static boolean getBoolean(String key) {
return Boolean.parseBoolean(getP(key, "false"));
}
public static int getInt(String key, int def) {
return Integer.parseInt(getP(key, Integer.toString(def)));
}
public static int getInt(String key) {
return Integer.parseInt(getP(key, "0"));
}
public static long getLong(String key, long def) {
return Long.parseLong(getP(key, Long.toString(def)));
}
public static long getLong(String key) {
return Long.parseLong(getP(key, "0L"));
}
public static double getDouble(String key, double def) {
return Double.parseDouble(getP(key, Double.toString(def)));
}
public static double getDouble(String key) {
return Double.parseDouble(getP(key, "0.0"));
}
private static String getP(String key, String def) {
return Utils.resolve(allProps.getProperty(key, def));
}
static Properties getProperties() {
return p;
}
static Properties getMergedProperties() {
return allProps;
}
private static void put(String key, String value) {
p.setProperty(key, value);
}
/**
* Called when the properties are updated.
*
*/
private static void rehashProperties() {
allProps.clear();
allProps.putAll(Config.properties());
allProps.putAll(System.getProperties());
Map<String, String> env = System.getenv();
for (String key : env.keySet()) {
String newKey = key.replace("___", "-").replace("__", ".");
allProps.put(newKey, env.get(key));
}
}
}