Config.java [src/csip] Revision: Date:
/*
* $Id: 2.8+8 Config.java 1ff5534b17ce 2023-11-28 od $
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2022, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip;
import csip.api.client.ModelDataServiceCall;
import csip.ModelDataService.Task;
import csip.utils.Binaries;
import csip.utils.SimpleCache;
import java.io.File;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.servlet.ServletContext;
/**
* Global properties, adjustable at runtime.
*
* @author od
*/
public class Config {
private static final Properties p = new Properties();
private static final Properties allProps = new Properties();
private static final List<Task> tasks = Collections.synchronizedList(new ArrayList<Task>());
//
// static private SimpleCache<String, AutoCloseable> backends = new SimpleCache<>();
//
private static ExecutorService exec;
private static Timer timer;
private static final Registry reg = new Registry();
static final Logger LOG = Logger.getLogger("csip-core");
static {
LOG.setLevel(Level.OFF);
}
private static final Object timerLock = new Object();
/**
* CSIP API version. Set by the system. <br>
*/
public static final String CSIP_PLATFORM_VERSION = "csip.platform.version";
public static final String CSIP_CONTEXT_VERSION = "csip.context.version";
/**
* CSIP platform Architecture. Set by the system.<br>
* e.g.: lin-amd46, win-x86,...
*/
public static final String CSIP_ARCH = "csip.arch";
/**
* Remote Access ACL. list of IPs or subnets that are allows to connect via
* UI. This ACL is not for the services, just service management. <br>
* default: "127.0.0.1/32" (localhost only)
*/
public static final String CSIP_REMOTE_ACL = "csip.remote.acl";
public static final String CSIP_TIMEZONE = "csip.timezone";
public static final String CSIP_LOGGING_LEVEL = "csip.logging.level";
public static final String CSIP_RESPONSE_STACKTRACE = "csip.response.stacktrace";
public static final String CSIP_RESPONSE_SQLFILTER = "csip.response.sqlfilter";
public static final String CSIP_RESPONSE_JSONINDENT = "csip.response.jsonindent";
public static final String CSIP_RESPONSE_ERROR_HTTPSTATUS = "csip.response.error.httpstatus";
public static final String CSIP_PAYLOAD_VERSION = "csip.payload.version";
//
public static final String CSIP_SESSION_BACKEND = "csip.session.backend";
public static final String CSIP_SESSION_TTL = "csip.session.ttl";
public static final String CSIP_SESSION_MONGODB_URI = "csip.session.mongodb.uri";
public static final String CSIP_SESSION_TTL_FAILED = "csip.session.ttl.failed";
public static final String CSIP_SESSION_TTL_CANCELLED = "csip.session.ttl.cancelled";
public static final String CSIP_ARCHIVE_BACKEND = "csip.archive.backend";
public static final String CSIP_ARCHIVE_MONGODB_URI = "csip.archive.mongodb.uri";
public static final String CSIP_ARCHIVE_MAX_FILE_SIZE = "csip.archive.max.filesize";
public static final String CSIP_ARCHIVE_TTL = "csip.archive.ttl";
public static final String CSIP_ARCHIVE_FAILEDONLY = "csip.archive.failedonly";
// public static final String CSIP_ARCHIVE_ONREQUEST = "csip.archive.onrequest";
public static final String CSIP_RESULTSTORE_BACKEND = "csip.resultstore.backend";
public static final String CSIP_RESULTSTORE_MONGODB_URI = "csip.resultstore.mongodb.uri";
public static final String CSIP_RESULTSTORE_LIMIT = "csip.resultstore.limit";
public static final String CSIP_ACCESSLOG_BACKEND = "csip.accesslog.backend";
public static final String CSIP_ACCESSLOG_MONGODB_URI = "csip.accesslog.mongodb.uri";
public static final String CSIP_ACCESSLOG_MONGODB_COL = "csip.accesslog.mongodb.col";
//
public static final String CSIP_DIR = "csip.dir";
public static final String CSIP_BIN_DIR = "csip.bin.dir";
public static final String CSIP_WORK_DIR = "csip.work.dir";
public static final String CSIP_RESULTS_DIR = "csip.results.dir";
public static final String CSIP_CACHE_DIR = "csip.cache.dir";
public static final String CSIP_DATA_DIR = "csip.data.dir";
//
public static final String CSIP_SNAPSHOT = "csip.snapshot";
public static final String CSIP_KEEPWORKSPACE = "csip.keepworkspace";
public static final String CSIP_JDBC_CHECKVALID = "csip.jdbc.checkvalid";
// values for session/archive/result
public static final String MONGODB = "mongodb";
public static final String LOCAL = "local";
public static final String SQL = "sql";
public static final String NONE = "none";
// public static final String KAFKA = "kafka";
public static final String WEBHOOK = "webhook";
// publisher
public static final String CSIP_PUBLISHER_BACKEND = "csip.publisher.backend";
// public static final String CSIP_PUBLISHER_KAFKA_BOOTSTRAP_SERVERS = "csip.publisher.kafka.bootstrap_servers";
// public static final String CSIP_PUBLISHER_KAFKA_ACKS = "csip.publisher.kafka.acks";
// public static final String CSIP_PUBLISHER_KAFKA_TOPIC = "csip.publisher.kafka.topic";
// public static final String CSIP_PUBLISHER_KAFKA_RETRIES = "csip.publisher.kafka.retries.";
// public static final String CSIP_PUBLISHER_KAFKA_MAX_BLOCK_MS = "csip.publisher.kafka.max_block_ms.";
// Auth
public static final String CSIP_TOKEN_AUTHENTICATION = "csip.token.authentication";
// internal services
static final String CSIP_PUBSUB_ENABLED = "csip.pubsub.enabled";
static final String CSIP_DYNPY_ENABLED = "csip.dynpy.enabled";
private static final Backends be = new Backends(LOG);
public synchronized static Map<Object, Object> properties() {
return Collections.unmodifiableMap(p);
}
//
static SimpleCache<File, ReentrantLock> wsFileLocks = new SimpleCache<>();
public static Logger getSystemLogger(ModelDataServiceCall.ConfAccess a) {
if (a == null)
throw new NullPointerException("Illegal Access.");
return LOG;
}
private synchronized static void setDefaultProperties() {
try {
/*
* The CSIP version for platform and context (placeholder)
*/
put(CSIP_PLATFORM_VERSION, "$version: 2.8.29 default 922 bb648022bf50 2024-10-10 od, built at 2024-11-14 09:11 by od$");
put(CSIP_CONTEXT_VERSION, "$version: 0.0.0 000000000000");
put("csip.response.version", "true");
// put("csip.internal.uri", "https://129.82.10.125");
/*
* The runtime architecture.
*/
put(CSIP_ARCH, Binaries.getArch());
// for legacy settings only.
put("arch", Binaries.getArch());
// remote access acl
/*
* remote access for UI and config. Provide a list of IPs or subnets that
* are allows to connect. This ACL is not for the services, just service
* management. The default is localhost only.
*
* Example: "csip.remoteaccess.acl": "127.0.0.1/32 10.2.222.0/24"
*/
put(CSIP_REMOTE_ACL, "127.0.0.1 0:0:0:0:0:0:0:1");
// session
/*
* The backend to use for session management. valid choices are "mongodb",
* "sql", "local".
*/
put(CSIP_SESSION_BACKEND, LOCAL);
/*
* The mongodb connecion string.
*/
put(CSIP_SESSION_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
* The default time in seconds for a session to stay active after the
* model finishes. All model results will be available for that period.
* After this period expires the session will be removed and the model
* results will be removed or archived. This value can be altered using
* the "keep_results" metainfo value of a request.
*
* see duration string examples:
* https://docs.oracle.com/javase/8/docs/api/java/time/Duration.html#parse-java.lang.CharSequence-
*/
put(CSIP_SESSION_TTL, "PT30S"); // 30 sec ttl
put(CSIP_SESSION_TTL_FAILED, "PT30S");
put(CSIP_SESSION_TTL_CANCELLED, "PT30S");
/*
* The default csip timezone to be used for time management.
*/
put(CSIP_TIMEZONE, "MST7MDT");
/*
* Use http status code for the response, default is 200. If changed, this
* value should be > 500, otherwise it defaults to 200. If set to default
* and a service results in an error, the POST response will return a 200
* status and the payload with the error message.
*
* If set to a value, e.g. "530" and a service results in an error, the
* POST response will return a 530 status and the payload with the error
* message. In addition, "400" will be returned if input is malformed or
* empty. Choose your code wisely!
*/
put(CSIP_RESPONSE_ERROR_HTTPSTATUS, "200");
/*
* Vary the payload sructure, JSONArray vs JSONObject for
* parameter/results entries. Defaults to version 1 (Array). JSONArray:
* "1" JSONObject: "2"
*/
put(CSIP_PAYLOAD_VERSION, "1");
// Usage Backend
put(CSIP_ACCESSLOG_BACKEND, NONE);
put(CSIP_ACCESSLOG_MONGODB_URI, "mongodb://localhost:27017/csip");
put(CSIP_ACCESSLOG_MONGODB_COL, "usage");
// archive
/*
* defines the archive backend implementation: "mongodb" or "none" are
* possible. "none" means disabled.
*/
put(CSIP_ARCHIVE_BACKEND, NONE);
/*
* The mongodb connection uri, if the backend is set to "mongodb"
*/
put(CSIP_ARCHIVE_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
* The max file size for an attachment to be archived.
*/
put(CSIP_ARCHIVE_MAX_FILE_SIZE, "10MB");
/*
* The default time in seconds for an entry to stay in the archive. All
* archived model results will be available for that period after the
* session expired. After this period expires the archive will be removed.
* * see duration string examples:
* https://docs.oracle.com/javase/8/docs/api/java/time/Duration.html#parse-java.lang.CharSequence-.
*/
put(CSIP_ARCHIVE_TTL, "P1D"); // one day.
/*
* If the archive is enabled, only archive failed runs, default: false
*/
put(CSIP_ARCHIVE_FAILEDONLY, "false");
//
// /*
// * set to true if archiving should be done only if requested
// */
// put(CSIP_ARCHIVE_ONREQUEST, "false");
///// Publish
/*
* NONE or KAFKA
*
*/
put(CSIP_PUBLISHER_BACKEND, NONE);
// Authentication
/**
* NONE
*/
put(CSIP_TOKEN_AUTHENTICATION, NONE);
//// Logging
/*
* The log level for service and system logging. . All
* java.util.logging.Level log levels are usable.
*/
put(CSIP_LOGGING_LEVEL, "CONFIG");
/*
* control if the stack trace should be part of the response metadata if
* the logic fails. default is false. Enable this for debugging.
*/
put(CSIP_RESPONSE_STACKTRACE, "false");
/*
* control if SQL error messages should be masked in the response payload.
* default is true.
*/
put(CSIP_RESPONSE_SQLFILTER, "true");
/*
* indentation of the response json, default is 2. set it to 0 to minify
* the response json.
*/
put(CSIP_RESPONSE_JSONINDENT, "2");
/*
* Result store handling
*/
put(CSIP_RESULTSTORE_BACKEND, NONE);
put(CSIP_RESULTSTORE_MONGODB_URI, "mongodb://localhost:27017/csip");
/*
* The csip root directory
*/
put(CSIP_DIR, "/tmp/csip");
/*
* The csip directories for executables.
*/
put(CSIP_BIN_DIR, "${csip.dir}/bin");
/*
* The csip directory for sessions.
*/
put(CSIP_WORK_DIR, "${csip.dir}/work");
/*
* The csip directory to store results.
*/
put(CSIP_RESULTS_DIR, "${csip.dir}/results");
/*
* The csip cache file directory.
*/
put(CSIP_CACHE_DIR, "${csip.dir}/cache");
/*
* The csip data directory.
*/
put(CSIP_DATA_DIR, "${csip.dir}/data");
// Core services as needed. need to be enabled in context config.s.
put(CSIP_DYNPY_ENABLED, "false");
put(CSIP_PUBSUB_ENABLED, "false");
/*
* External url pats These properties can be set to force a public
* scheme/host/protocol for result file downloads and catalog listing.
* These properties are not set per default. They can be set independently
* from each other to change only selective parts of the URL. If none of
* the propeties below are set the incomming URL is being used to
* construct downloads and catalogs.
*/
// put("csip.public.scheme", ""); // e.g. https
// put("csip.public.host", ""); // e.g. csip.org
// put("csip.public.port", ""); // e.g. 8080 (-1 will remove the port in the url)
///////////////////////////////////////////////////////
//// more auxiliary properties
// thread management
put("codebase.threadpool", "32"); // 10 concurrent model runs.
put("codebase.url", "http://localhost:8080");
put("codebase.port", "8085");
put("codebase.localport", "8081");
put("codebase.servicename", "csip-vmscaler");
// wine
put("wine.path", "/usr/bin/wine");
//rusle2/weps
// put("r2.path", "/od/projects/csip.services/bin/RomeShell.exe"); // the path is the parent directory.
// put("r2.db", "http://oms-db.engr.colostate.edu/r2");
// put("weps.db", "http://oms-db.engr.colostate.edu/weps");
//oms related props
// put("oms.java.home", "/usr");
put("oms.java.home", "/opt/jdk1.8.0_51");
put("oms.esp.threads", "4");
// internal
put("vm.port", "8080");
} catch (Exception E) {
Logger.getLogger(Config.class.getName()).log(Level.SEVERE, "Static init error", E);
}
}
private static String getVersion(String version) {
if (version == null || version.isEmpty())
return "?";
String v[] = version.split("\\s+");
if (v.length < 3)
return "?";
return v[1] + "(" + v[2] + ")";
}
private static String full_version;
static synchronized String getFullVersion() {
if (full_version == null)
full_version = getVersion(getString(CSIP_CONTEXT_VERSION)) + "-"
+ getVersion(getString(CSIP_PLATFORM_VERSION));
return full_version;
}
public static void register(Set<Class<?>> services, ServletContext context) {
ContextConfig.filterServices(context, services);
getRegistry().register(services);
}
static Registry getRegistry() {
return reg;
}
static boolean isArchiveEnabled() {
return !getString(CSIP_ARCHIVE_BACKEND, NONE).equals(NONE);
}
static boolean isResultStoreEnabled() {
return !getString(CSIP_RESULTSTORE_BACKEND, NONE).equals(NONE);
}
static SessionStore getSessionStore() {
return be.getSessionStore();
}
static ArchiveStore getArchiveStore() {
return be.getArchiveStore();
}
static AccessLog getAccessLog() {
return be.getAccessLog();
}
static Publisher getPublisher() {
return be.getPublisher();
}
static ResultStore getResultStore() {
return be.getResultStore();
}
static TokenAuthentication getTokenAuthentication() {
return be.getTokenAuthentication();
}
static Timer getTimer() {
synchronized (timerLock) {
if (timer == null)
timer = new Timer();
return timer;
}
}
/**
* This is being called if schedule results in an illegal state exception.
*
* @return
*/
static Timer getNewTimer() {
synchronized (timerLock) {
LOG.info("Starting new timer for Session TTL handling.");
timer = new Timer();
return timer;
}
}
static synchronized ExecutorService getExecutorService() {
if (exec == null)
exec = Executors.newCachedThreadPool();
return exec;
}
static List<ModelDataService.Task> getModelTasks() {
return tasks;
}
static private void setupLogging() {
Level l = Level.parse(getString(CSIP_LOGGING_LEVEL));
LOG.log(Level.INFO, " LogLevel : {0}", l.toString());
Logger rootLogger = Logger.getLogger("");
rootLogger.setLevel(l);
for (Handler h : rootLogger.getHandlers()) {
h.setLevel(l);
}
LOG.setLevel(l);
}
/**
* Start up the servlet.
*
* @param context
*/
static void startup() {
setupLogging();
// getSessionStore().registerResources(true);
}
/**
* Shut down the servlet.
*
* @param context
*/
static void shutdown() {
tasks.forEach(task -> task.cancel());
reg.unregister();
if (exec != null) {
LOG.info("Shutting down ExecutorService");
exec.shutdownNow();
}
be.closeAll();
// session.registerResources(false);
if (timer != null)
timer.cancel();
Binaries.shutdownJDBC();
}
public static boolean hasProperty(String key) {
return allProps.containsKey(key);
}
public static boolean isString(String key, String str) {
String s = getString(key);
return (s != null) && s.equals(str);
}
public static String getString(String key, String def) {
return getP(key, def);
}
public static String getString(String key) {
return getP(key, null);
}
public static boolean getBoolean(String key, boolean def) {
return Boolean.parseBoolean(getP(key, Boolean.toString(def)));
}
public static boolean getBoolean(String key) {
return Boolean.parseBoolean(getP(key, "false"));
}
public static int getInt(String key, int def) {
return Integer.parseInt(getP(key, Integer.toString(def)));
}
public static int getInt(String key) {
return Integer.parseInt(getP(key, "0"));
}
public static long getLong(String key, long def) {
return Long.parseLong(getP(key, Long.toString(def)));
}
public static long getLong(String key) {
return Long.parseLong(getP(key, "0L"));
}
public static double getDouble(String key, double def) {
return Double.parseDouble(getP(key, Double.toString(def)));
}
public static double getDouble(String key) {
return Double.parseDouble(getP(key, "0.0"));
}
private static String getP(String key, String def) {
return Utils.resolve(allProps.getProperty(key, def));
}
static Properties getProperties() {
if (!p.containsKey(CSIP_PLATFORM_VERSION))
setDefaultProperties();
return p;
}
static Properties getMergedProperties() {
return allProps;
}
private static void put(String key, String value) {
p.setProperty(key, value);
}
/**
* Called when the properties are updated.
*
*/
static void update() {
allProps.clear();
allProps.putAll(Config.properties());
allProps.putAll(System.getProperties());
Map<String, String> env = System.getenv();
for (String key : env.keySet()) {
String newKey = key.replace("___", "-").replace("__", ".");
allProps.put(newKey, env.get(key));
}
}
}