core: fix #60: Add --dupfile option to log dupes

Author: nil0x42

.gitignore

@@ -1,5 +1,6 @@
 /duplicut
 /nonreg_*.out
+/nonregdupes_*.out
 /objects
 *.gcno
 *.gcda

include/config.h

@@ -6,6 +6,7 @@ struct              conf
 {
     const char      *infile_name;
     const char      *outfile_name;
+    const char      *dupfile_name;
     unsigned int    threads;
     unsigned int    line_max_size;
     size_t          hmap_size;

include/const.h

@@ -4,6 +4,10 @@
 #define PROJECT_VERSION         "2.3"
 #define PROJECT_URL             "http://github.com/nil0x42/duplicut"
 
+/* cast a preprocessor int to string */
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
 /* portion of available memory to allow to hashmap */
 #define HMAP_MAX_SIZE           (0.5)
 
@@ -24,3 +28,6 @@
 
 /* display program status periodically instead of waiting keypress */
 #define DEBUG_PROGRAM_STATUS    (0)
+
+/* max supported max-line-size */
+#define MAX_MAX_LINE_SIZE       4095

include/file.h

@@ -15,5 +15,6 @@ struct file
 extern struct file  *g_file;
 
 /* source file: file.c */
-void    init_file(const char *infile_name, const char *outfile_name);
-void    destroy_file(void);
+void    init_files(void);
+void    cleanup_files(void);
+void    log_duplicate(char *ln, int sz);

src/chunk.c

@@ -103,6 +103,7 @@ void        cleanout_chunk(t_chunk *chunk)
     char    *base_ptr;
     int     i;
     long    duplicates;
+    const int has_dupfile = (g_conf.dupfile_name != NULL);
 
     duplicates = 0;
     i = 0;
@@ -115,6 +116,9 @@ void        cleanout_chunk(t_chunk *chunk)
         {
             if (cmp_line(&line, &g_hmap.ptr[slot]) == 0)
             {
+                if (has_dupfile) {
+                    log_duplicate(LINE_ADDR(line), LINE_SIZE(line));
+                }
                 LINE_ADDR(line)[0] = DISABLED_LINE;
                 ++duplicates;
                 break;

src/config.c

@@ -128,6 +128,11 @@ void            config(void)
     DLOG1("---------- g_conf ------------");
     DLOG1("g_conf.infile_name:       %s", g_conf.infile_name);
     DLOG1("g_conf.outfile_name:      %s", g_conf.outfile_name);
+    if (g_conf.dupfile_name != NULL) {
+        DLOG1("g_conf.dupfile_name:      %s", g_conf.dupfile_name);
+    } else {
+        DLOG1("g_conf.dupfile_name:      (null)");
+    }
     DLOG1("g_conf.threads:           %u", g_conf.threads);
     DLOG1("g_conf.line_max_size:     %u", g_conf.line_max_size);
     DLOG1("g_conf.hmap_size:         %s (%ld slots of %dbits)",

src/file.c

@@ -7,6 +7,7 @@
 #include <sys/types.h>
 #include <ctype.h>
 #include "file.h"
+#include "line.h"
 #include "const.h"
 #include "status.h"
 #include "error.h"
@@ -21,12 +22,15 @@
 static struct file  g_infile;
 static struct file  g_outfile;
 static struct file  g_tmpfile;
+static struct file  g_dupfile;
 
 
 /** Ensure that infile and outfile don't point to the same
  * file on filesystem, by comparing device & inode.
  */
-static void check_files(const char *infile_name, const char *outfile_name)
+static void check_files_difer(
+        const char *infile_name,
+        const char *outfile_name)
 {
     struct stat     infile_info;
     struct stat     outfile_info;
@@ -62,6 +66,7 @@ static void close_all(void)
 {
     close_file(&g_infile);
     close_file(&g_outfile);
+    close_file(&g_dupfile);
 
     if (FILE_ISSET(&g_tmpfile))
     {
@@ -175,19 +180,24 @@ static void copy_file(int dst_fd, int src_fd, bool send_status)
  * - Registers cleanup functions with atexit()
  * - returned file has `addr` attribute mapped in memory
  */
-void        init_file(const char *infile_name, const char *outfile_name)
+void        init_files(void)
 {
     struct file  *file;
 
-    check_files(infile_name, outfile_name);
+    check_files_difer(g_conf.infile_name, g_conf.outfile_name);
 
     memset(&g_infile, -1, sizeof(g_infile));
-    memset(&g_outfile, -1, sizeof(g_infile));
-    memset(&g_tmpfile, -1, sizeof(g_infile));
+    memset(&g_outfile, -1, sizeof(g_outfile));
+    memset(&g_tmpfile, -1, sizeof(g_tmpfile));
+    memset(&g_dupfile, -1, sizeof(g_dupfile));
 
     atexit(close_all);
-    open_file(&g_infile, infile_name, O_RDONLY);
-    open_file(&g_outfile, outfile_name, O_TRUNC | O_CREAT | O_RDWR);
+    open_file(&g_infile, g_conf.infile_name, O_RDONLY);
+    open_file(&g_outfile, g_conf.outfile_name, O_TRUNC | O_CREAT | O_RDWR);
+
+    if (g_conf.dupfile_name != NULL) {
+        open_file(&g_dupfile, g_conf.dupfile_name, O_TRUNC | O_CREAT | O_RDWR);
+    }
 
     if (S_ISREG(g_outfile.info.st_mode))
         file = &g_outfile;
@@ -230,7 +240,7 @@ void        init_file(const char *infile_name, const char *outfile_name)
  * - truncate file with new size.
  * - Call close_all() for cleanout.
  */
-void        destroy_file(void)
+void        cleanup_files(void)
 {
     struct file     *file;
 
@@ -252,3 +262,19 @@ void        destroy_file(void)
 
     close_all();
 }
+
+/** Write a duplicate to dupfile_name (must be atomic)
+ */
+void        log_duplicate(char *line_ptr, int line_sz)
+{
+    ssize_t n;
+    char buf[MAX_MAX_LINE_SIZE + 1];
+    memcpy(buf, line_ptr, line_sz);
+    buf[line_sz] = '\n';
+    do {
+        n = write(g_dupfile.fd, buf, line_sz + 1);
+    } while (n == -1 && errno == EINTR);
+    if (n == -1) {
+        error("log_duplicate() -> write(): %s", ERRNO);
+    }
+}

src/hmap.c

@@ -1,7 +1,10 @@
 #include <stdlib.h>
+#include "file.h"
 #include "hmap.h"
 #include "hash.h"
+#include "line.h"
 #include "const.h"
+#include "config.h"
 #include "status.h"
 #include "error.h"
 #include "debug.h"
@@ -51,6 +54,7 @@ void        populate_hmap(t_chunk *chunk)
     char    *base_ptr;
     int     i;
     long    duplicates;
+    const int has_dupfile = (g_conf.dupfile_name != NULL);
 
     duplicates = 0;
     i = 0;
@@ -94,6 +98,9 @@ void        populate_hmap(t_chunk *chunk)
             }
             else if (cmp_line(&line, &g_hmap.ptr[slot]) == 0)
             {
+                if (has_dupfile) {
+                    log_duplicate(LINE_ADDR(line), LINE_SIZE(line));
+                }
                 LINE_ADDR(line)[0] = DISABLED_LINE;
                 ++duplicates;
                 break;

src/main.c

@@ -15,6 +15,7 @@
 struct conf g_conf = {
     .infile_name = NULL,
     .outfile_name = NULL,
+    .dupfile_name = NULL,
     .threads = 0,
     .line_max_size = DEFAULT_LINE_MAX_SIZE,
     .hmap_size = 0,
@@ -76,7 +77,7 @@ int             main(int argc, char **argv)
         watch_user_input();
 
     update_status(FCOPY_START);
-    init_file(g_conf.infile_name, g_conf.outfile_name);
+    init_files();
     config(); /* configure g_conf options */
     set_status(CHUNK_SIZE, g_conf.chunk_size);
 
@@ -87,7 +88,7 @@ int             main(int argc, char **argv)
 
     update_status(FCLEAN_START);
     remove_duplicates();
-    destroy_file();
+    cleanup_files();
 
     display_report();
     return (0);

src/optparse.c

@@ -14,7 +14,7 @@
 
 /** Arguments configuration for getopt_long().
  */
-#define OPTSTRING "o:t:m:l:pcChv"
+#define OPTSTRING "o:t:m:l:pcCD:hv"
 
 static struct option    g_options[] = {
     { "outfile",       required_argument, NULL, 'o' },
@@ -24,6 +24,7 @@ static struct option    g_options[] = {
     { "printable",     no_argument,       NULL, 'p' },
     { "lowercase",     no_argument,       NULL, 'c' },
     { "uppercase",     no_argument,       NULL, 'C' },
+    { "dupfile",       required_argument, NULL, 'D' },
     { "help",          no_argument,       NULL, 'h' },
     { "version",       no_argument,       NULL, 'v' },
     { NULL,            0,                 NULL, '\0'},
@@ -63,6 +64,14 @@ static void setopt_outfile(const char *value)
     g_conf.outfile_name = value;
 }
 
+/** Set dup file (-D option) [OPTIONAL]
+ * --> The destination file
+ */
+static void setopt_dupfile(const char *value)
+{
+    g_conf.dupfile_name = value;
+}
+
 
 static void setopt_threads(const char *value)
 {
@@ -109,9 +118,9 @@ static void setopt_line_max_size(const char *value)
     {
         bad_argument("line_max_size", value, "not a positive number");
     }
-    else if (line_max_size > 4095)
+    else if (line_max_size > MAX_MAX_LINE_SIZE)
     {
-        bad_argument("line_max_size", value, "max value is 4095");
+        bad_argument("line_max_size", value, "max value is " STR(MAX_MAX_LINE_SIZE));
     }
     g_conf.line_max_size = (unsigned int) line_max_size;
 }
@@ -155,6 +164,7 @@ static void setopt_help(const char *value)
            "-p, --printable            Filter ascii printable lines\n"
            "-c, --lowercase            Convert wordlist to lowercase\n"
            "-C, --uppercase            Convert wordlist to uppercase\n"
+           "-D, --dupfile <FILE>       Write dupes to <FILE> (slows down duplicut)\n"
            "-h, --help                 Display this help and exit\n"
            "-v, --version              Output version information and exit\n"
            "\n"
@@ -193,6 +203,7 @@ static void setopt(int opt, const char *value)
         { 'p', setopt_printable },
         { 'c', setopt_lowercase },
         { 'C', setopt_uppercase },
+        { 'D', setopt_dupfile },
         { 'h', setopt_help },
         { 'v', setopt_version },
         { '\0', NULL }