[PATCH] RFC: impliment gzipped output for notmuch dump
Jani Nikula
jani at nikula.org
Sat Mar 29 02:25:12 PDT 2014
On Sat, 29 Mar 2014, David Bremner <david at tethera.net> wrote:
> ---
>
> This obviously needs at least one piece of tidying, and some tests.
>
> My motivation here is the ability to make gzipped backups from within
> notmuch, e.g. as part of an upgrade procedure.
Is it reasonable to require zlib? It's possible to have conditional
build for this, but it will be laborous to do it cleanly. I could be
persuaded either way, perhaps more easily to just requiring it.
If you do require zlib, the missing dependencies error message from
configure should include a check for have_zlib and a note about it
similar to other required dependencies.
Also, it would seem natural to support gzipped input in restore as
well... do you think it would be silly to support gzipped output only
internally until we've added gzipped input too? This could be in a later
release in the future.
BR,
Jani.
>
>
> Makefile.local | 2 +-
> configure | 19 ++++++++++++++++++-
> notmuch-client.h | 4 +++-
> notmuch-dump.c | 41 +++++++++++++++++++++++++++--------------
> 4 files changed, 49 insertions(+), 17 deletions(-)
>
> diff --git a/Makefile.local b/Makefile.local
> index cb7b106..e5a20a7 100644
> --- a/Makefile.local
> +++ b/Makefile.local
> @@ -41,7 +41,7 @@ PV_FILE=bindings/python/notmuch/version.py
> # Smash together user's values with our extra values
> FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)
> FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)
> -FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)
> +FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS)
> FINAL_NOTMUCH_LINKER = CC
> ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
> FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)
> diff --git a/configure b/configure
> index 1d430b9..02ef785 100755
> --- a/configure
> +++ b/configure
> @@ -340,6 +340,18 @@ else
> errors=$((errors + 1))
> fi
>
> +printf "Checking for zlib development files... "
> +have_zlib=0
> +if pkg-config --exists zlib; then
> + printf "Yes.\n"
> + have_zlib=1
> + zlib_cflags=$(pkg-config --cflags zlib)
> + zlib_ldflags=$(pkg-config --libs zlib)
> +else
> + printf "No.\n"
> + errors=$((errors + 1))
> +fi
> +
> printf "Checking for talloc development files... "
> if pkg-config --exists talloc; then
> printf "Yes.\n"
> @@ -844,6 +856,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags}
> GMIME_CFLAGS = ${gmime_cflags}
> GMIME_LDFLAGS = ${gmime_ldflags}
>
> +# Flags needed to compile and link against zlib
> +ZLIB_CFLAGS = ${zlib_cflags}
> +ZLIB_LDFLAGS = ${zlib_ldflags}
> +
> # Flags needed to compile and link against talloc
> TALLOC_CFLAGS = ${talloc_cflags}
> TALLOC_LDFLAGS = ${talloc_ldflags}
> @@ -882,6 +898,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
> -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
>
> CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
> + \$(ZLIB_CFLAGS) \\
> \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\
> \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\
> -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\
> @@ -892,5 +909,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\
> -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\
> -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
>
> -CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS)
> +CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)
> EOF
> diff --git a/notmuch-client.h b/notmuch-client.h
> index d110648..e1efbe0 100644
> --- a/notmuch-client.h
> +++ b/notmuch-client.h
> @@ -450,7 +450,9 @@ typedef enum dump_formats {
> int
> notmuch_database_dump (notmuch_database_t *notmuch,
> const char *output_file_name,
> - const char *query_str, dump_format_t output_format);
> + const char *query_str,
> + dump_format_t output_format,
> + notmuch_bool_t gzip_output);
>
> #include "command-line-arguments.h"
> #endif
> diff --git a/notmuch-dump.c b/notmuch-dump.c
> index 21702d7..029f90a 100644
> --- a/notmuch-dump.c
> +++ b/notmuch-dump.c
> @@ -21,10 +21,12 @@
> #include "notmuch-client.h"
> #include "hex-escape.h"
> #include "string-util.h"
> +#include <zlib.h>
> +
>
> static int
> -database_dump_file (notmuch_database_t *notmuch, FILE *output,
> - const char *query_str, int output_format)
> +database_dump_file (notmuch_database_t *notmuch, gzFile output,
> + const char *query_str, int output_format)
> {
> notmuch_query_t *query;
> notmuch_messages_t *messages;
> @@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
> }
>
> if (output_format == DUMP_FORMAT_SUP) {
> - fprintf (output, "%s (", message_id);
> + gzprintf (output, "%s (", message_id);
> }
>
> for (tags = notmuch_message_get_tags (message);
> @@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
> const char *tag_str = notmuch_tags_get (tags);
>
> if (! first)
> - fputs (" ", output);
> + gzputs (output, " ");
>
> first = 0;
>
> if (output_format == DUMP_FORMAT_SUP) {
> - fputs (tag_str, output);
> + gzputs (output, tag_str);
> } else {
> if (hex_encode (notmuch, tag_str,
> &buffer, &buffer_size) != HEX_SUCCESS) {
> @@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
> tag_str);
> return EXIT_FAILURE;
> }
> - fprintf (output, "+%s", buffer);
> + gzprintf (output, "+%s", buffer);
> }
> }
>
> if (output_format == DUMP_FORMAT_SUP) {
> - fputs (")\n", output);
> + gzputs (output, ")\n");
> } else {
> if (make_boolean_term (notmuch, "id", message_id,
> &buffer, &buffer_size)) {
> @@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
> message_id, strerror (errno));
> return EXIT_FAILURE;
> }
> - fprintf (output, " -- %s\n", buffer);
> + gzprintf (output, " -- %s\n", buffer);
> }
>
> notmuch_message_destroy (message);
> @@ -121,24 +123,33 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
> int
> notmuch_database_dump (notmuch_database_t *notmuch,
> const char *output_file_name,
> - const char *query_str, dump_format_t output_format)
> + const char *query_str,
> + dump_format_t output_format,
> + notmuch_bool_t gzip_output)
> {
> - FILE *output = stdout;
> + gzFile output;
> + const char *mode = gzip_output ? "w9" : "wT";
> +
> int ret;
>
> if (output_file_name) {
> - output = fopen (output_file_name, "w");
> + output = gzopen (output_file_name, mode);
> if (output == NULL) {
> fprintf (stderr, "Error opening %s for writing: %s\n",
> output_file_name, strerror (errno));
> return EXIT_FAILURE;
> }
> + } else {
> + output = gzdopen (fileno (stdout), mode);
> }
>
> ret = database_dump_file (notmuch, output, query_str, output_format);
>
> - if (output != stdout)
> - fclose (output);
> + /* XXX check error return */
> + gzflush (output, Z_FINISH);
> +
> + if (output_file_name)
> + gzclose_w (output);
>
> return ret;
> }
> @@ -158,6 +169,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
> int opt_index;
>
> int output_format = DUMP_FORMAT_BATCH_TAG;
> + notmuch_bool_t gzip_output = 0;
>
> notmuch_opt_desc_t options[] = {
> { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
> @@ -165,6 +177,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
> { "batch-tag", DUMP_FORMAT_BATCH_TAG },
> { 0, 0 } } },
> { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 },
> + { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
> { 0, 0, 0, 0, 0 }
> };
>
> @@ -181,7 +194,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
> }
>
> ret = notmuch_database_dump (notmuch, output_file_name, query_str,
> - output_format);
> + output_format, gzip_output);
>
> notmuch_database_destroy (notmuch);
>
> --
> 1.9.0
>
> _______________________________________________
> notmuch mailing list
> notmuch at notmuchmail.org
> http://notmuchmail.org/mailman/listinfo/notmuch
--
Jani
More information about the notmuch
mailing list