Displaying differences for changeset
 
display as  

src/java/m/timeseries/query/V1_0.java

@@ -11,6 +11,7 @@
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Aggregates;
 import com.mongodb.client.model.Filters;
 import javax.ws.rs.Path;
 import oms3.annotations.*;
@@ -24,7 +25,10 @@
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
 import java.util.TimeZone;
 import org.bson.Document;
@@ -73,6 +77,7 @@
         String start_date = getStringParam("start_date", null);
         String end_date = getStringParam("end_date", null);
         boolean metadata_only = getBooleanParam("metadata_only", true);
+        boolean compute_stats = getBooleanParam("compute_stats", false);
         String search_id = getStringParam("search_id", null);
         JSONObject search_feature;
         try {
@@ -80,19 +85,23 @@
         } catch (ServiceException ex) {
             throw new RuntimeException(ex);
         }
-        run(mongo_uri, mongo_collection, search_feature, search_id, start_date, end_date, metadata_only, LOG);
+        run(mongo_uri, mongo_collection, search_feature, search_id, start_date, end_date, metadata_only, compute_stats, LOG);
     }
     
     public void run(String mongo_uri, String mongo_collection, JSONObject search_feature, String search_id,
-            String start_date_str, String end_date_str, boolean metadata_only, SessionLogger LOG) throws JSONException {
+            String start_date_str, String end_date_str, boolean metadata_only, boolean compute_stats, SessionLogger LOG) throws JSONException, ServiceException {
         open(mongo_uri);
         MongoCollection<Document> collection = db.getCollection(mongo_collection);
 
-        LocalDate start_date = null;
-        LocalDate end_date = null;
-        if (start_date_str != null) {
-            start_date = LocalDate.parse(start_date_str);
-            end_date = LocalDate.parse(end_date_str);
+        SimpleDateFormat ft = new SimpleDateFormat("yyyy-MM-dd");
+        ft.setTimeZone(TimeZone.getTimeZone("UTC"));
+        Date start_date;
+        Date end_date;
+        try {
+            start_date = ft.parse(start_date_str);
+            end_date = ft.parse(end_date_str);
+        } catch (ParseException ex) {
+            throw new ServiceException(ex);
         }
 
         if (search_feature == null) {
@@ -110,7 +119,7 @@
                 }
             }
             for (Document doc: c) {
-                JSONObject res = getResult(doc, start_date, end_date, metadata_only);
+                JSONObject res = getResult(collection, doc, start_date, end_date, metadata_only, compute_stats);
                 results.put(res);
             }
         } else {
@@ -126,35 +135,92 @@
                 }
                 AggregateIterable<Document> c = collection.aggregate(agg_match);
                 for (Document doc: c) {
-                    JSONObject res = getResult(doc, start_date, end_date, metadata_only);
+                    JSONObject res = getResult(collection, doc, start_date, end_date, metadata_only, compute_stats);
                     results.put(res);
                 }
             }
         }
     }
 
-    private JSONObject getResult(Document doc, LocalDate start_date, LocalDate end_date, boolean metadata_only) throws JSONException {
+    private JSONObject getResult(MongoCollection<Document> c, Document doc, Date start_date, Date end_date,
+            boolean metadata_only, boolean compute_stats) throws JSONException {
         String date_fmt = (String)doc.get("date_fmt");
         SimpleDateFormat ft = new SimpleDateFormat(date_fmt);
         ft.setTimeZone(TimeZone.getTimeZone("UTC"));
 
         List<String> header = (ArrayList)doc.get("header");
-        List<List<Object>> data = (ArrayList)doc.get("data");
         JSONArray output = new JSONArray();
         List<List<Object>> filtered = new ArrayList();
+        JSONObject statResult = new JSONObject();
 
         if (!metadata_only) {
-            // Filter by period and format date
-            for (int irow=0; irow<data.size(); irow++) {
-                Date dt = (Date)data.get(irow).get(0);
-                LocalDate dt_local = dt.toInstant().atZone(ZoneId.of("UTC")).toLocalDate();
-                if (start_date != null && dt_local.isBefore(start_date)) {
-                    continue;
-                } else if (end_date != null && dt_local.isAfter(end_date)) {
-                    break;
+            Document datesQuery = new Document();
+            if ((start_date != null || end_date != null)) {
+                datesQuery = new Document();
+                if (start_date != null) { 
+                    datesQuery.append("$gte", start_date);
                 }
-                data.get(irow).set(0, ft.format(dt));
-                filtered.add(data.get(irow));
+                if (end_date != null) { 
+                    datesQuery.append("$lte", end_date);
+                }            
+                datesQuery = new Document("data.0", datesQuery);
+            }
+
+            Document project = new Document(
+                "date", new Document("$arrayElemAt", Arrays.asList("$data", 0))
+            );
+            for (int i=1; i<header.size(); i++) {
+                project.append(header.get(i), new Document("$arrayElemAt", Arrays.asList("$data", i)));
+            }
+        
+            Document group = new Document("_id", "big").append("data", new Document("$push", "$data"));
+
+            // Export for robomongo
+            List<Bson> aggList = Arrays.asList(
+                Aggregates.match(new Document("_id", doc.getString("_id"))),
+                Aggregates.unwind("$data"),
+                //Aggregates.project(project)
+                Aggregates.match(datesQuery),
+                // Wind array back
+                new Document("$group", group)
+            );
+            
+            AggregateIterable<Document> iter = c.aggregate(aggList).allowDiskUse(true);
+            for (Document d: iter) {
+                filtered = (List<List<Object>>) d.get("data");
+                // Fix date formatting
+                for (List<Object> objList: filtered) {
+                    Date dt = (Date)objList.get(0);
+                    objList.set(0, ft.format(dt));
+                }
+            }
+            
+            
+            if (compute_stats) {
+                Document statsDoc = new Document("_id", null);
+                List<String> stats = Arrays.asList("min", "max", "avg", "stdDevPop");
+                for (String stat: stats) {
+                    for (int i=1; i<header.size(); i++) {
+                        statsDoc.append(header.get(i) + "_" + stat, new Document("$" + stat, "$" + header.get(i)));
+                    }
+                }
+
+                aggList = Arrays.asList(
+                    Aggregates.match(new Document("_id", doc.getString("_id"))),
+                    Aggregates.unwind("$data"),
+                    Aggregates.match(datesQuery),
+                    Aggregates.project(project),
+                    new Document("$group", statsDoc)
+                );
+                iter = c.aggregate(aggList).allowDiskUse(true);
+                for (Document d: iter) {
+                    for (String stat: stats) {
+                        for (int i=1; i<header.size(); i++) {
+                            String key = header.get(i) + "_" + stat;
+                            statResult.put(key, d.get(key));
+                        }
+                    }
+                }
             }
         }
   
@@ -170,6 +236,11 @@
         Document metadata = (Document)doc.get("metadata");
         res.put("metadata", new JSONObject(metadata.toJson()));
         
+        if (compute_stats) {
+            res.put("statistics", statResult);
+        }
+        
+        
         return res;
     }
 
@@ -202,7 +273,7 @@
         return agg_match;
     }
     
-   public static void main(String [] args) throws FileNotFoundException, IOException, JSONException, ParseException {
+   public static void main(String [] args) throws FileNotFoundException, IOException, JSONException, ParseException, ServiceException {
         V1_0 timeseries = new V1_0();
         JSONObject searchFeature = new JSONObject("{\n" +
 "                \"type\": \"FeatureCollection\",\n" +
@@ -247,6 +318,6 @@
 "            }");
         SessionLogger LOG = new SessionLogger();
         String search_id = null;
-        timeseries.run("mongodb://eds0.engr.colostate.edu:27017/csip_timeseries", "test_coll", searchFeature, search_id, "2014-04-04", "2014-05-08", false, LOG);
+        timeseries.run("mongodb://eds0.engr.colostate.edu:27017/csip_timeseries", "test_coll", searchFeature, search_id, "2014-04-04", "2014-05-08", false, true, LOG);
     }
 }

src/java/m/timeseries/query/V1_0.json

@@ -23,6 +23,10 @@
             "description": "Limit search to the given id. Optional.",
             "value": "test_ref"
         }, {
+            "name": "compute stats",
+            "description": "Return min, max, mean of data.",
+            "value": false
+        }, {
             "name": "search_feature",
             "description": "Limit search to a polygon. If not provided, then all locations are returned.",
             "value": {