Lscountv2

cmd/bosun/expr/logstash.go

@@ -164,8 +164,9 @@ func (e *LogstashElasticHosts) GenIndices(r *LogstashRequest) (string, error) {
 // filter is an Elastic regexp query that can be applied to any field. It is in
 // the same format as the keystring argument.
 // interval is in the format of an opentsdb time duration, and tells elastic
-// what the bucket size should be. The result will be normalized to a per second
-// rate regardless of what this is set to.
+// what the bucket size should be.
+// Note: The results used to be normalized to a per second rate. This is no
+// longer done as it resulted in erroneous extrapolations.
 // sduration and end duration are the time bounds for the query and are in
 // opentsdb's relative time format:
 // http://opentsdb.net/docs/build/html/user_guide/query/dates.html
@@ -191,19 +192,16 @@ func LSStat(e *State, T miniprofiler.Timer, index_root, keystring, filter, field
 	return LSDateHistogram(e, T, index_root, keystring, filter, interval, sduration, eduration, field, rstat, 0)
 }
 
-// LSDateHistorgram builds the aggregation query using subaggregations. The result is a grouped timer series
+// LSDateHistorgram builds the aggregation query using subaggregations. The result is a grouped time series
 // that Bosun can understand
 func LSDateHistogram(e *State, T miniprofiler.Timer, index_root, keystring, filter, interval, sduration, eduration, stat_field, rstat string, size int) (r *Results, err error) {
 	r = new(Results)
 	req, err := LSBaseQuery(e.now, index_root, e.logstashHosts, keystring, filter, sduration, eduration, size)
 	if err != nil {
 		return nil, err
 	}
-	ts := elastic.NewDateHistogramAggregation().Field("@timestamp").Interval(strings.Replace(interval, "M", "n", -1)).MinDocCount(0)
-	ds, err := opentsdb.ParseDuration(interval)
-	if err != nil {
-		return nil, err
-	}
+	// Extended bounds and min doc count are required to get values back when the bucket value is 0
+	ts := elastic.NewDateHistogramAggregation().Field("@timestamp").Interval(strings.Replace(interval, "M", "n", -1)).MinDocCount(0).ExtendedBoundsMin(req.Start).ExtendedBoundsMax(req.End)
 	if stat_field != "" {
 		ts = ts.SubAggregation("stats", elastic.NewExtendedStatsAggregation().Field(stat_field))
 		switch rstat {
@@ -224,7 +222,7 @@ func LSDateHistogram(e *State, T miniprofiler.Timer, index_root, keystring, filt
 		}
 		series := make(Series)
 		for _, v := range ts.Buckets {
-			val := processBucketItem(v, rstat, ds)
+			val := processBucketItem(v, rstat)
 			if val != nil {
 				series[time.Unix(v.Key/1000, 0).UTC()] = *val
 			}
@@ -261,7 +259,7 @@ func LSDateHistogram(e *State, T miniprofiler.Timer, index_root, keystring, filt
 			}
 			series := make(Series)
 			for _, v := range ts.Buckets {
-				val := processBucketItem(v, rstat, ds)
+				val := processBucketItem(v, rstat)
 				if val != nil {
 					series[time.Unix(v.Key/1000, 0).UTC()] = *val
 				}
@@ -305,7 +303,7 @@ func LSDateHistogram(e *State, T miniprofiler.Timer, index_root, keystring, filt
 	return r, nil
 }
 
-func processBucketItem(b *elastic.AggregationBucketHistogramItem, rstat string, ds opentsdb.Duration) *float64 {
+func processBucketItem(b *elastic.AggregationBucketHistogramItem, rstat string) *float64 {
 	if stats, found := b.ExtendedStats("stats"); found {
 		var val *float64
 		switch rstat {
@@ -326,7 +324,7 @@ func processBucketItem(b *elastic.AggregationBucketHistogramItem, rstat string,
 		}
 		return val
 	}
-	v := float64(b.DocCount) / ds.Seconds()
+	v := float64(b.DocCount)
 	return &v
 }
 

docs/expressions.md

@@ -148,14 +148,16 @@ q("sum:2m-avg:rate{counter,,1}:os.cpu{host=*}", "30m", "")
 
 ### lscount(indexRoot string, keyString string, filterString string, bucketDuration string, startDuration string, endDuration string) seriesSet
 
-lscount returns the per second rate of matching log documents.
+lscount returns a time bucked count of matching log documents.
 
   * `indexRoot` is the root name of the index to hit, the format is expected to be `fmt.Sprintf("%s-%s", index_root, d.Format("2006.01.02"))`.
   * `keyString` creates groups (like tagsets) and can also filter those groups. It is the format of `"field:regex,field:regex..."` The `:regex` can be ommited.
   * `filterString` is an Elastic regexp query that can be applied to any field. It is in the same format as the keystring argument.
-  * `bucketDuration` is in the same format is an opentsdb duration, and is the size of buckets returned (i.e. counts for every 10 minutes). In the case of lscount, that number is normalized to a per second rate by dividing the result by the number of seconds in the duration.
+  * `bucketDuration` is in the same format is an opentsdb duration, and is the size of buckets returned (i.e. counts for every 10 minutes). 
   * `startDuration` and `endDuration` set the time window from now - see the OpenTSDB q() function for more details.
 
+**Note:** As of Bosun 0.5.0, the results are no longer normalized per second. This resulted in bad extrapolations, and confusing interactions with functions like `sum(lscount(...))`. The rate will now be per bucket. If you still want the results normalized to per second, you can divide the result by the number of seconds with: `lscount("logstash", "logsource,program:bosun", $bucketDuration, "10m", "") / d($bucketDuration)`
+
 For example:
 
 `lscount("logstash", "logsource,program:bosun", "5s", "10m", "")`