https://lists.balabit.hu/pipermail/syslog-ng/2009-August/013325.html [syslog-ng] PATCH: Log file size limit (was: Logfile Rotation) Balazs Scheidler wrote: >> >> Personally I prefer to arrange log rotation based on the file size >> rather than some fixed time interval, so I have added that feature >> to syslog-ng. If anyone is interested I can explain in more details >> and post the patch. > > Please do. That's what open source is about. > The following patch introduces two new configuration options: 'file_size_limit' to be used inside global 'options' and 'size_limit' to be used inside 'file' destination. Each option specifies log file size limit in bytes. If the global option is set to a value greater than zero it applies to all 'file' destinations. A particular file destination can remove the limit by setting it to zero. For example: # set the global file size limit options { file_size_limit(123456); }; # set a different size limit for a particular file destination destination log1 { file("/var/log/log1.log" size_limit(456789)); }; # remove size limit for a particular file destination # (only useful if there is a global size limit set) destination log2 { file("/var/log/log2.log" size_limit(0)); }; The file size is checked after writing each log message and if the file has grown up to or above the size limit the file is renamed and a new empty file is created to continue logging to. The name format the "overgrown" log file is renamed to is "

-.-", where

- the full path and name of the original log file, - current time in UNIX format (seconds since Jan 1, 1970), - fractional part of the current time (microseconds, 6 digits), - a random number (10 digits). The intended use is to have incrond or another similar mechanism to detect when there is a new "renamed" log file and to process it in whatever way necessary (gzip it, parse it, send it my email, etc.). *********** BEGIN PATCH *********** diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.c syslog-ng-3.0.4/src/affile.c --- syslog-ng-3.0.4-orig/src/affile.c 2009-07-31 11:41:20.000000000 +0200 +++ syslog-ng-3.0.4/src/affile.c 2009-08-17 11:02:41.000000000 +0200 @@ -428,14 +428,72 @@ { return log_queue_get_length(((LogWriter *) self->writer)->queue) == 0; } static gboolean +affile_dw_open_file(AFFileDestWriter *self, int *fd) +{ + return affile_open_file(self->filename->str, (self->owner->flags & AFFILE_PIPE)? + (O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE): + (O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE), + self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, + self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, + !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd); +} + +static void +affile_dw_transport_callback(gint fd, void *context) +{ + static const size_t extra_length = 32; + AFFileDestWriter *self = (AFFileDestWriter *) context; + off_t size = lseek(fd, 0, SEEK_CUR); + GTimeVal time; + GString *name; + int reopen_fd; + if ((size > 0) && (size >= self->owner->size_limit)) + { + /* TODO: use g_file_read_link() */ + g_get_current_time(&time); + name = g_string_sized_new(self->filename->len + extra_length); + g_string_printf(name, "%s-%lu.%06lu-%010lu", self->filename->str, (gulong)time.tv_sec, (gulong)time.tv_usec, (gulong)g_random_int()); + if (rename(self->filename->str, name->str) == 0) + { + if (affile_dw_open_file(self, &reopen_fd)) + { + if (dup2(reopen_fd, fd) < 0) + { + msg_error("Error swithing to new log file", + evt_tag_str("filename", self->filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + close(reopen_fd); + } + else + { + msg_error("Error opening file for writing", + evt_tag_str("filename", self->filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + } + else + { + msg_error("Error renaming overgrown file", + evt_tag_str("filename", self->filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + g_string_free(name, TRUE); + } +} + +static gboolean affile_dw_init(LogPipe *s) { AFFileDestWriter *self = (AFFileDestWriter *) s; - int fd, flags; + int fd; struct stat st; GlobalConfig *cfg = log_pipe_get_config(s); if (cfg) self->time_reopen = cfg->time_reopen; @@ -454,20 +512,12 @@ evt_tag_int("overwrite_if_older", self->owner->overwrite_if_older), NULL); unlink(self->filename->str); } - if (self->owner->flags & AFFILE_PIPE) - flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - else - flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - self->last_open_stamp = time(NULL); - if (affile_open_file(self->filename->str, flags, - self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, - self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, - !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd)) + if (affile_dw_open_file(self, &fd)) { guint write_flags; if (!self->writer) { @@ -482,11 +532,11 @@ self->writer = NULL; close(fd); return FALSE; } write_flags = ((self->owner->flags & AFFILE_FSYNC) ? LTF_FSYNC : 0) | LTF_APPEND; - log_writer_reopen(self->writer, log_proto_plain_new_client(log_transport_plain_new(fd, write_flags))); + log_writer_reopen(self->writer, log_proto_plain_new_client(((self->owner->size_limit > 0) && !(self->owner->flags & AFFILE_PIPE))? log_transport_plain_new_with_callback(fd, write_flags, affile_dw_transport_callback, self): log_transport_plain_new(fd, write_flags))); } else { msg_error("Error opening file for writing", evt_tag_str("filename", self->filename->str), @@ -678,10 +728,18 @@ AFFileDestDriver *self = (AFFileDestDriver *) s; self->local_time_zone = g_strdup(local_time_zone); } +void +affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit) +{ + AFFileDestDriver *self = (AFFileDestDriver *) s; + + self->size_limit = file_size_limit; +} + static inline gchar * affile_dd_format_persist_name(AFFileDestDriver *self) { static gchar persist_name[1024]; @@ -766,10 +824,12 @@ self->dir_gid = cfg->dir_gid; if (self->dir_perm == (mode_t) -1) self->dir_perm = cfg->dir_perm; if (self->time_reap == -1) self->time_reap = cfg->time_reap; + if (self->size_limit == -1) + self->size_limit = cfg->file_size_limit; self->use_time_recvd = cfg->use_time_recvd; if (self->local_time_zone_info) time_zone_info_free(self->local_time_zone_info); @@ -973,7 +1033,8 @@ if (strchr(filename, '$') == NULL) { self->flags |= AFFILE_NO_EXPAND; } self->time_reap = -1; + self->size_limit = -1; return &self->super; } diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.h syslog-ng-3.0.4/src/affile.h --- syslog-ng-3.0.4-orig/src/affile.h 2009-04-30 12:22:53.000000000 +0200 +++ syslog-ng-3.0.4/src/affile.h 2009-08-17 10:10:39.000000000 +0200 @@ -72,10 +72,11 @@ gint overwrite_if_older; gboolean use_time_recvd; gint time_reap; guint reap_timer; + off_t size_limit; } AFFileDestDriver; LogDriver *affile_dd_new(gchar *filename, guint32 flags); void affile_dd_set_compress(LogDriver *s, gboolean compress); @@ -88,7 +89,8 @@ void affile_dd_set_dir_perm(LogDriver *s, mode_t dir_perm); void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs); void affile_dd_set_fsync(LogDriver *s, gboolean enable); void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older); void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone); +void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit); #endif diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-grammar.y syslog-ng-3.0.4/src/cfg-grammar.y --- syslog-ng-3.0.4-orig/src/cfg-grammar.y 2009-08-05 12:34:18.000000000 +0200 +++ syslog-ng-3.0.4/src/cfg-grammar.y 2009-08-17 10:10:39.000000000 +0200 @@ -154,10 +154,11 @@ %token KW_DIR_OWNER KW_DIR_GROUP KW_DIR_PERM %token KW_TEMPLATE KW_TEMPLATE_ESCAPE %token KW_FOLLOW_FREQ %token KW_OVERWRITE_IF_OLDER %token KW_DEFAULT_FACILITY KW_DEFAULT_LEVEL +%token KW_FILE_SIZE_LIMIT KW_SIZE_LIMIT /* socket related options */ %token KW_KEEP_ALIVE KW_MAX_CONNECTIONS %token KW_LOCALIP KW_IP KW_LOCALPORT KW_PORT KW_DESTPORT %token KW_IP_TTL KW_SO_BROADCAST KW_IP_TOS KW_SO_SNDBUF KW_SO_RCVBUF KW_SO_KEEPALIVE KW_SPOOF_SOURCE @@ -799,10 +800,11 @@ | KW_DIR_PERM '(' LL_NUMBER ')' { affile_dd_set_dir_perm(last_driver, $3); } | KW_CREATE_DIRS '(' yesno ')' { affile_dd_set_create_dirs(last_driver, $3); } | KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')' { affile_dd_set_overwrite_if_older(last_driver, $3); } | KW_FSYNC '(' yesno ')' { affile_dd_set_fsync(last_driver, $3); } | KW_LOCAL_TIME_ZONE '(' string ')' { affile_dd_set_local_time_zone(last_driver, $3); free($3); } + | KW_SIZE_LIMIT '(' LL_NUMBER ')' { affile_dd_set_file_size_limit(last_driver, $3); } ; dest_afpipe : KW_PIPE '(' dest_afpipe_params ')' { $$ = $3; } ; @@ -1141,10 +1143,11 @@ | KW_FILE_TEMPLATE '(' string ')' { configuration->file_template_name = g_strdup($3); free($3); } | KW_PROTO_TEMPLATE '(' string ')' { configuration->proto_template_name = g_strdup($3); free($3); } | KW_RECV_TIME_ZONE '(' string ')' { configuration->recv_time_zone = g_strdup($3); free($3); } | KW_SEND_TIME_ZONE '(' string ')' { configuration->send_time_zone = g_strdup($3); free($3); } | KW_LOCAL_TIME_ZONE '(' string ')' { configuration->local_time_zone = g_strdup($3); free($3); } + | KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')' { configuration->file_size_limit = $3; } ; /* BEGIN MARK: tls */ tls_options : tls_option tls_options diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-lex.l syslog-ng-3.0.4/src/cfg-lex.l --- syslog-ng-3.0.4-orig/src/cfg-lex.l 2009-05-06 11:20:22.000000000 +0200 +++ syslog-ng-3.0.4/src/cfg-lex.l 2009-08-17 10:10:39.000000000 +0200 @@ -126,10 +126,12 @@ { "recv_time_zone", KW_RECV_TIME_ZONE }, { "send_time_zone", KW_SEND_TIME_ZONE }, { "local_time_zone", KW_LOCAL_TIME_ZONE }, { "use_time_recvd", KW_USE_TIME_RECVD, KWS_OBSOLETE, "Use R_ or S_ prefixed macros in templates" }, { "use_fqdn", KW_USE_FQDN }, + { "size_limit", KW_SIZE_LIMIT }, + { "file_size_limit", KW_FILE_SIZE_LIMIT }, { "use_dns", KW_USE_DNS }, { "gc_threshold", KW_GC_BUSY_THRESHOLD }, { "gc_busy_threshold", KW_GC_BUSY_THRESHOLD }, { "gc_idle_threshold", KW_GC_IDLE_THRESHOLD }, { "time_reopen", KW_TIME_REOPEN }, diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.c syslog-ng-3.0.4/src/cfg.c --- syslog-ng-3.0.4-orig/src/cfg.c 2009-04-30 12:00:54.000000000 +0200 +++ syslog-ng-3.0.4/src/cfg.c 2009-08-17 10:10:39.000000000 +0200 @@ -298,10 +298,11 @@ self->follow_freq = -1; self->file_uid = 0; self->file_gid = 0; self->file_perm = 0600; + self->file_size_limit = 0; self->dir_uid = 0; self->dir_gid = 0; self->dir_perm = 0700; self->use_dns_cache = 1; diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.h syslog-ng-3.0.4/src/cfg.h --- syslog-ng-3.0.4-orig/src/cfg.h 2009-06-03 13:08:27.000000000 +0200 +++ syslog-ng-3.0.4/src/cfg.h 2009-08-17 10:10:39.000000000 +0200 @@ -78,10 +78,11 @@ gint follow_freq; gboolean create_dirs; uid_t file_uid; gid_t file_gid; mode_t file_perm; + off_t file_size_limit; uid_t dir_uid; gid_t dir_gid; mode_t dir_perm; diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.c syslog-ng-3.0.4/src/logtransport.c --- syslog-ng-3.0.4-orig/src/logtransport.c 2009-04-30 10:53:22.000000000 +0200 +++ syslog-ng-3.0.4/src/logtransport.c 2009-08-17 11:01:33.000000000 +0200 @@ -53,10 +53,12 @@ typedef struct _LogTransportPlain LogTransportPlain; struct _LogTransportPlain { LogTransport super; + LogTransportCallback callback; + void *context; }; static gssize log_transport_plain_read_method(LogTransport *s, gpointer buf, gsize buflen, GSockAddr **sa) { @@ -138,24 +140,32 @@ alarm_cancel(); if (self->super.flags & LTF_FSYNC) fsync(self->super.fd); } while (rc == -1 && errno == EINTR); + if ((self->callback != NULL) && (rc > 0)) + self->callback(self->super.fd, self->context); return rc; } LogTransport * -log_transport_plain_new(gint fd, guint flags) +log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context) { LogTransportPlain *self = g_new0(LogTransportPlain, 1); self->super.fd = fd; self->super.cond = 0; self->super.flags = flags; self->super.read = log_transport_plain_read_method; self->super.write = log_transport_plain_write_method; self->super.free_fn = log_transport_free_method; + self->callback = callback; + self->context = context; return &self->super; } - +LogTransport * +log_transport_plain_new(gint fd, guint flags) +{ + return log_transport_plain_new_with_callback(fd, flags, NULL, NULL); +} diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.h syslog-ng-3.0.4/src/logtransport.h --- syslog-ng-3.0.4-orig/src/logtransport.h 2009-04-30 10:46:55.000000000 +0200 +++ syslog-ng-3.0.4/src/logtransport.h 2009-08-17 11:00:58.000000000 +0200 @@ -57,10 +57,12 @@ log_transport_read(LogTransport *self, gpointer buf, gsize count, GSockAddr **sa) { return self->read(self, buf, count, sa); } +typedef void (*LogTransportCallback)(gint fd, void *context); +LogTransport *log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context); LogTransport *log_transport_plain_new(gint fd, guint flags); void log_transport_free(LogTransport *s); void log_transport_free_method(LogTransport *s);