Merge branch 'tureus-docker-updates' into versions/0.10

CHANGES.txt

@@ -20,6 +20,10 @@ Bug Handling
 
 * Force ElasticSearch index name to lower case, as required by ElasticSearch.
 
+* Fixed default buffer config values.
+
+* Updated docker configuration so docker build works again (#1634).
+
 0.10.0b1 (2015-08-07)
 =====================
 

Dockerfile

@@ -1,5 +1,5 @@
 # heka_base image
-FROM debian:jessie
+FROM golang:1.4
 
 MAINTAINER Chance Zibolski <chance.zibolski@gmail.com> (@chance)
 
@@ -18,27 +18,15 @@ RUN     apt-get update && \
         ruby-dev \
         protobuf-compiler \
         python-sphinx \
-        wget
-
-# Install Go 1.3.1
-RUN curl -s https://storage.googleapis.com/golang/go1.3.1.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
-        echo "3af011cc19b21c7180f2604fd85fbc4ddde97143 /tmp/go.tar.gz" | sha1sum -c && \
-        tar -C /usr/local -xzf /tmp/go.tar.gz
+        wget \
+        debhelper \
+        fakeroot \
+        libgeoip-dev \
+        libgeoip1 \
+        golang-goprotobuf-dev
 
 WORKDIR /heka
 
-ENV GOROOT /usr/local/go
-ENV PATH $PATH:/usr/local/go/bin:/go/bin
-
-ENV CTEST_OUTPUT_ON_FAILURE 1
-ENV BUILD_DIR   /heka/build
-ENV GOPATH      $BUILD_DIR/heka
-ENV GOBIN       $GOPATH/bin
-ENV PATH        $PATH:$GOBIN
-# Build faster
-ENV NUM_JOBS    10
-
 EXPOSE 4352
 
 COPY . /heka
-RUN ./build.sh

docker/Dockerfile

@@ -3,8 +3,10 @@
 # and builds a new image which installs that dpkg
 FROM mozilla/heka_base
 
+RUN cd /heka && . ./build.sh
+RUN cd /heka && . ./env.sh && cd /heka/build && make deb
+
 RUN mkdir -p /heka_docker
-RUN cd /heka/build && make deb
 RUN find /heka/build -name "*.deb" -exec cp {} /heka_docker/heka.deb \;
 COPY Dockerfile.final /heka_docker/Dockerfile
 
@@ -13,4 +15,4 @@ RUN curl -sSL https://get.docker.io/builds/Linux/x86_64/docker-1.2.0 -o /tmp/doc
     mv /tmp/docker /usr/local/bin/docker && \
     chmod +x /usr/local/bin/docker
 
-CMD docker build -t mozilla/heka /heka_docker
+CMD docker build -t mozilla/heka /heka_docker

docker/Dockerfile.final

@@ -4,7 +4,8 @@ FROM debian:jessie
 MAINTAINER Chance Zibolski <chance.zibolski@gmail.com> (@chance)
 
 COPY heka.deb /tmp/heka.deb
+RUN apt-get update && apt-get install -y libgeoip1
 RUN dpkg -i /tmp/heka.deb && rm /tmp/heka.deb
 
 EXPOSE 4352
-ENTRYPOINT ["hekad"]
+ENTRYPOINT ["hekad"]

docs/source/buffering.rst

@@ -23,7 +23,7 @@ Buffering configuration settings
 - max_file_size (uint64)
   The maximum size (in bytes) of a single file in the queue buffer. When a
   message would increase a queue file to greater than this size, the message
-  will be written into a new file instead. Defaults to 128MiB. Value cannot
+  will be written into a new file instead. Defaults to 512MiB. Value cannot
   be zero, if zero is specified the default will instead be used.
 
 - max_buffer_size (uint64)

pipeline/config.go

@@ -490,11 +490,11 @@ func getAttr(ob interface{}, attr string, default_ interface{}) (ret interface{}
 	obVal = reflect.Indirect(obVal) // Dereference if it's a pointer.
 	if obVal.Kind().String() != "struct" {
 		// `FieldByName` will panic if we're not a struct.
-		return
+		return ret
 	}
 	attrVal := obVal.FieldByName(attr)
 	if !attrVal.IsValid() {
-		return
+		return ret
 	}
 	return attrVal.Interface()
 }

pipeline/plugin_maker.go

@@ -427,8 +427,9 @@ func (m *pluginMaker) MakeRunner(name string) (PluginRunner, error) {
 			return nil, err
 		}
 	}
+
 	if commonFO.Buffering == nil {
-		bufConfig := getAttr(config, "Buffering", &QueueBufferConfig{})
+		bufConfig := getAttr(config, "Buffering", defaultQueueBufferConfig())
 		switch c := bufConfig.(type) {
 		case *QueueBufferConfig:
 			commonFO.Buffering = c

pipeline/queue_buffer.go

@@ -66,6 +66,17 @@ type QueueBufferConfig struct {
 	CursorUpdateCount uint   `toml:"cursor_update_count"`
 }
 
+const DefaultBufferMaxFileSize uint64 = uint64(512 * 1024 * 1024)
+
+func defaultQueueBufferConfig() *QueueBufferConfig {
+	return &QueueBufferConfig{
+		MaxFileSize:       DefaultBufferMaxFileSize,
+		MaxBufferSize:     uint64(0),
+		FullAction:        "shutdown",
+		CursorUpdateCount: uint(1),
+	}
+}
+
 func NewBufferSet(queueDir, queueName string, config *QueueBufferConfig,
 	runner *foRunner, pConfig *PipelineConfig) (*BufferFeeder, *BufferReader, error) {
 
@@ -87,6 +98,9 @@ func NewBufferSet(queueDir, queueName string, config *QueueBufferConfig,
 		config.CursorUpdateCount = 1
 	}
 
+	if config.MaxFileSize == 0 {
+		config.MaxFileSize = DefaultBufferMaxFileSize // 512 MiB
+	}
 	if config.MaxFileSize < uint64(message.MAX_RECORD_SIZE) {
 		err := fmt.Errorf("`max_file_size` must be greater than maximum record size of %d",
 			message.MAX_RECORD_SIZE)
@@ -118,9 +132,6 @@ type BufferFeeder struct {
 func NewBufferFeeder(queue string, config *QueueBufferConfig, queueSize *BufferSize) (
 	*BufferFeeder, error) {
 
-	if config.MaxFileSize == 0 {
-		config.MaxFileSize = 512 * 1024 * 1024 // 512 MiB
-	}
 	bf := &BufferFeeder{
 		queue:     queue,
 		queueSize: queueSize,