diff --git a/CHANGES.txt b/CHANGES.txt index 2487eb90f..a3da593a9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -20,6 +20,10 @@ Bug Handling * Force ElasticSearch index name to lower case, as required by ElasticSearch. +* Fixed default buffer config values. + +* Updated docker configuration so docker build works again (#1634). + 0.10.0b1 (2015-08-07) ===================== diff --git a/Dockerfile b/Dockerfile index a431bef48..a27e07b1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # heka_base image -FROM debian:jessie +FROM golang:1.4 MAINTAINER Chance Zibolski (@chance) @@ -18,27 +18,15 @@ RUN apt-get update && \ ruby-dev \ protobuf-compiler \ python-sphinx \ - wget - -# Install Go 1.3.1 -RUN curl -s https://storage.googleapis.com/golang/go1.3.1.linux-amd64.tar.gz -o /tmp/go.tar.gz && \ - echo "3af011cc19b21c7180f2604fd85fbc4ddde97143 /tmp/go.tar.gz" | sha1sum -c && \ - tar -C /usr/local -xzf /tmp/go.tar.gz + wget \ + debhelper \ + fakeroot \ + libgeoip-dev \ + libgeoip1 \ + golang-goprotobuf-dev WORKDIR /heka -ENV GOROOT /usr/local/go -ENV PATH $PATH:/usr/local/go/bin:/go/bin - -ENV CTEST_OUTPUT_ON_FAILURE 1 -ENV BUILD_DIR /heka/build -ENV GOPATH $BUILD_DIR/heka -ENV GOBIN $GOPATH/bin -ENV PATH $PATH:$GOBIN -# Build faster -ENV NUM_JOBS 10 - EXPOSE 4352 COPY . /heka -RUN ./build.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 53948772c..91b0cb31b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,8 +3,10 @@ # and builds a new image which installs that dpkg FROM mozilla/heka_base +RUN cd /heka && . ./build.sh +RUN cd /heka && . ./env.sh && cd /heka/build && make deb + RUN mkdir -p /heka_docker -RUN cd /heka/build && make deb RUN find /heka/build -name "*.deb" -exec cp {} /heka_docker/heka.deb \; COPY Dockerfile.final /heka_docker/Dockerfile @@ -13,4 +15,4 @@ RUN curl -sSL https://get.docker.io/builds/Linux/x86_64/docker-1.2.0 -o /tmp/doc mv /tmp/docker /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -CMD docker build -t mozilla/heka /heka_docker +CMD docker build -t mozilla/heka /heka_docker \ No newline at end of file diff --git a/docker/Dockerfile.final b/docker/Dockerfile.final index bccaff8c9..216ceecb3 100644 --- a/docker/Dockerfile.final +++ b/docker/Dockerfile.final @@ -4,7 +4,8 @@ FROM debian:jessie MAINTAINER Chance Zibolski (@chance) COPY heka.deb /tmp/heka.deb +RUN apt-get update && apt-get install -y libgeoip1 RUN dpkg -i /tmp/heka.deb && rm /tmp/heka.deb EXPOSE 4352 -ENTRYPOINT ["hekad"] +ENTRYPOINT ["hekad"] \ No newline at end of file diff --git a/docs/source/buffering.rst b/docs/source/buffering.rst index 2ee1517f6..25b056271 100644 --- a/docs/source/buffering.rst +++ b/docs/source/buffering.rst @@ -23,7 +23,7 @@ Buffering configuration settings - max_file_size (uint64) The maximum size (in bytes) of a single file in the queue buffer. When a message would increase a queue file to greater than this size, the message - will be written into a new file instead. Defaults to 128MiB. Value cannot + will be written into a new file instead. Defaults to 512MiB. Value cannot be zero, if zero is specified the default will instead be used. - max_buffer_size (uint64) diff --git a/pipeline/config.go b/pipeline/config.go index 6a83df83d..38d254f4a 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -490,11 +490,11 @@ func getAttr(ob interface{}, attr string, default_ interface{}) (ret interface{} obVal = reflect.Indirect(obVal) // Dereference if it's a pointer. if obVal.Kind().String() != "struct" { // `FieldByName` will panic if we're not a struct. - return + return ret } attrVal := obVal.FieldByName(attr) if !attrVal.IsValid() { - return + return ret } return attrVal.Interface() } diff --git a/pipeline/plugin_maker.go b/pipeline/plugin_maker.go index 0e44a19bf..12a91e6d2 100644 --- a/pipeline/plugin_maker.go +++ b/pipeline/plugin_maker.go @@ -427,8 +427,9 @@ func (m *pluginMaker) MakeRunner(name string) (PluginRunner, error) { return nil, err } } + if commonFO.Buffering == nil { - bufConfig := getAttr(config, "Buffering", &QueueBufferConfig{}) + bufConfig := getAttr(config, "Buffering", defaultQueueBufferConfig()) switch c := bufConfig.(type) { case *QueueBufferConfig: commonFO.Buffering = c diff --git a/pipeline/queue_buffer.go b/pipeline/queue_buffer.go index 264d882d9..cec2e0bcc 100644 --- a/pipeline/queue_buffer.go +++ b/pipeline/queue_buffer.go @@ -66,6 +66,17 @@ type QueueBufferConfig struct { CursorUpdateCount uint `toml:"cursor_update_count"` } +const DefaultBufferMaxFileSize uint64 = uint64(512 * 1024 * 1024) + +func defaultQueueBufferConfig() *QueueBufferConfig { + return &QueueBufferConfig{ + MaxFileSize: DefaultBufferMaxFileSize, + MaxBufferSize: uint64(0), + FullAction: "shutdown", + CursorUpdateCount: uint(1), + } +} + func NewBufferSet(queueDir, queueName string, config *QueueBufferConfig, runner *foRunner, pConfig *PipelineConfig) (*BufferFeeder, *BufferReader, error) { @@ -87,6 +98,9 @@ func NewBufferSet(queueDir, queueName string, config *QueueBufferConfig, config.CursorUpdateCount = 1 } + if config.MaxFileSize == 0 { + config.MaxFileSize = DefaultBufferMaxFileSize // 512 MiB + } if config.MaxFileSize < uint64(message.MAX_RECORD_SIZE) { err := fmt.Errorf("`max_file_size` must be greater than maximum record size of %d", message.MAX_RECORD_SIZE) @@ -118,9 +132,6 @@ type BufferFeeder struct { func NewBufferFeeder(queue string, config *QueueBufferConfig, queueSize *BufferSize) ( *BufferFeeder, error) { - if config.MaxFileSize == 0 { - config.MaxFileSize = 512 * 1024 * 1024 // 512 MiB - } bf := &BufferFeeder{ queue: queue, queueSize: queueSize,