@@ -11,6 +11,7 @@ |
import com.mongodb.client.FindIterable; |
import com.mongodb.client.MongoCollection; |
import com.mongodb.client.MongoDatabase; |
+import com.mongodb.client.model.Aggregates; |
import com.mongodb.client.model.Filters; |
import javax.ws.rs.Path; |
import oms3.annotations.*; |
@@ -24,7 +25,10 @@ |
import java.time.LocalDate; |
import java.time.ZoneId; |
import java.util.ArrayList; |
+import java.util.Arrays; |
+import java.util.Calendar; |
import java.util.Date; |
+import java.util.GregorianCalendar; |
import java.util.List; |
import java.util.TimeZone; |
import org.bson.Document; |
@@ -73,6 +77,7 @@ |
String start_date = getStringParam("start_date", null); |
String end_date = getStringParam("end_date", null); |
boolean metadata_only = getBooleanParam("metadata_only", true); |
+ boolean compute_stats = getBooleanParam("compute_stats", false); |
String search_id = getStringParam("search_id", null); |
JSONObject search_feature; |
try { |
@@ -80,19 +85,23 @@ |
} catch (ServiceException ex) { |
throw new RuntimeException(ex); |
} |
- run(mongo_uri, mongo_collection, search_feature, search_id, start_date, end_date, metadata_only, LOG); |
+ run(mongo_uri, mongo_collection, search_feature, search_id, start_date, end_date, metadata_only, compute_stats, LOG); |
} |
|
public void run(String mongo_uri, String mongo_collection, JSONObject search_feature, String search_id, |
- String start_date_str, String end_date_str, boolean metadata_only, SessionLogger LOG) throws JSONException { |
+ String start_date_str, String end_date_str, boolean metadata_only, boolean compute_stats, SessionLogger LOG) throws JSONException, ServiceException { |
open(mongo_uri); |
MongoCollection<Document> collection = db.getCollection(mongo_collection); |
|
- LocalDate start_date = null; |
- LocalDate end_date = null; |
- if (start_date_str != null) { |
- start_date = LocalDate.parse(start_date_str); |
- end_date = LocalDate.parse(end_date_str); |
+ SimpleDateFormat ft = new SimpleDateFormat("yyyy-MM-dd"); |
+ ft.setTimeZone(TimeZone.getTimeZone("UTC")); |
+ Date start_date; |
+ Date end_date; |
+ try { |
+ start_date = ft.parse(start_date_str); |
+ end_date = ft.parse(end_date_str); |
+ } catch (ParseException ex) { |
+ throw new ServiceException(ex); |
} |
|
if (search_feature == null) { |
@@ -110,7 +119,7 @@ |
} |
} |
for (Document doc: c) { |
- JSONObject res = getResult(doc, start_date, end_date, metadata_only); |
+ JSONObject res = getResult(collection, doc, start_date, end_date, metadata_only, compute_stats); |
results.put(res); |
} |
} else { |
@@ -126,35 +135,92 @@ |
} |
AggregateIterable<Document> c = collection.aggregate(agg_match); |
for (Document doc: c) { |
- JSONObject res = getResult(doc, start_date, end_date, metadata_only); |
+ JSONObject res = getResult(collection, doc, start_date, end_date, metadata_only, compute_stats); |
results.put(res); |
} |
} |
} |
} |
|
- private JSONObject getResult(Document doc, LocalDate start_date, LocalDate end_date, boolean metadata_only) throws JSONException { |
+ private JSONObject getResult(MongoCollection<Document> c, Document doc, Date start_date, Date end_date, |
+ boolean metadata_only, boolean compute_stats) throws JSONException { |
String date_fmt = (String)doc.get("date_fmt"); |
SimpleDateFormat ft = new SimpleDateFormat(date_fmt); |
ft.setTimeZone(TimeZone.getTimeZone("UTC")); |
|
List<String> header = (ArrayList)doc.get("header"); |
- List<List<Object>> data = (ArrayList)doc.get("data"); |
JSONArray output = new JSONArray(); |
List<List<Object>> filtered = new ArrayList(); |
+ JSONObject statResult = new JSONObject(); |
|
if (!metadata_only) { |
- // Filter by period and format date |
- for (int irow=0; irow<data.size(); irow++) { |
- Date dt = (Date)data.get(irow).get(0); |
- LocalDate dt_local = dt.toInstant().atZone(ZoneId.of("UTC")).toLocalDate(); |
- if (start_date != null && dt_local.isBefore(start_date)) { |
- continue; |
- } else if (end_date != null && dt_local.isAfter(end_date)) { |
- break; |
+ Document datesQuery = new Document(); |
+ if ((start_date != null || end_date != null)) { |
+ datesQuery = new Document(); |
+ if (start_date != null) { |
+ datesQuery.append("$gte", start_date); |
} |
- data.get(irow).set(0, ft.format(dt)); |
- filtered.add(data.get(irow)); |
+ if (end_date != null) { |
+ datesQuery.append("$lte", end_date); |
+ } |
+ datesQuery = new Document("data.0", datesQuery); |
+ } |
+ |
+ Document project = new Document( |
+ "date", new Document("$arrayElemAt", Arrays.asList("$data", 0)) |
+ ); |
+ for (int i=1; i<header.size(); i++) { |
+ project.append(header.get(i), new Document("$arrayElemAt", Arrays.asList("$data", i))); |
+ } |
+ |
+ Document group = new Document("_id", "big").append("data", new Document("$push", "$data")); |
+ |
+ // Export for robomongo |
+ List<Bson> aggList = Arrays.asList( |
+ Aggregates.match(new Document("_id", doc.getString("_id"))), |
+ Aggregates.unwind("$data"), |
+ //Aggregates.project(project) |
+ Aggregates.match(datesQuery), |
+ // Wind array back |
+ new Document("$group", group) |
+ ); |
+ |
+ AggregateIterable<Document> iter = c.aggregate(aggList).allowDiskUse(true); |
+ for (Document d: iter) { |
+ filtered = (List<List<Object>>) d.get("data"); |
+ // Fix date formatting |
+ for (List<Object> objList: filtered) { |
+ Date dt = (Date)objList.get(0); |
+ objList.set(0, ft.format(dt)); |
+ } |
+ } |
+ |
+ |
+ if (compute_stats) { |
+ Document statsDoc = new Document("_id", null); |
+ List<String> stats = Arrays.asList("min", "max", "avg", "stdDevPop"); |
+ for (String stat: stats) { |
+ for (int i=1; i<header.size(); i++) { |
+ statsDoc.append(header.get(i) + "_" + stat, new Document("$" + stat, "$" + header.get(i))); |
+ } |
+ } |
+ |
+ aggList = Arrays.asList( |
+ Aggregates.match(new Document("_id", doc.getString("_id"))), |
+ Aggregates.unwind("$data"), |
+ Aggregates.match(datesQuery), |
+ Aggregates.project(project), |
+ new Document("$group", statsDoc) |
+ ); |
+ iter = c.aggregate(aggList).allowDiskUse(true); |
+ for (Document d: iter) { |
+ for (String stat: stats) { |
+ for (int i=1; i<header.size(); i++) { |
+ String key = header.get(i) + "_" + stat; |
+ statResult.put(key, d.get(key)); |
+ } |
+ } |
+ } |
} |
} |
|
@@ -170,6 +236,11 @@ |
Document metadata = (Document)doc.get("metadata"); |
res.put("metadata", new JSONObject(metadata.toJson())); |
|
+ if (compute_stats) { |
+ res.put("statistics", statResult); |
+ } |
+ |
+ |
return res; |
} |
|
@@ -202,7 +273,7 @@ |
return agg_match; |
} |
|
- public static void main(String [] args) throws FileNotFoundException, IOException, JSONException, ParseException { |
+ public static void main(String [] args) throws FileNotFoundException, IOException, JSONException, ParseException, ServiceException { |
V1_0 timeseries = new V1_0(); |
JSONObject searchFeature = new JSONObject("{\n" + |
" \"type\": \"FeatureCollection\",\n" + |
@@ -247,6 +318,6 @@ |
" }"); |
SessionLogger LOG = new SessionLogger(); |
String search_id = null; |
- timeseries.run("mongodb://eds0.engr.colostate.edu:27017/csip_timeseries", "test_coll", searchFeature, search_id, "2014-04-04", "2014-05-08", false, LOG); |
+ timeseries.run("mongodb://eds0.engr.colostate.edu:27017/csip_timeseries", "test_coll", searchFeature, search_id, "2014-04-04", "2014-05-08", false, true, LOG); |
} |
} |