feat(import): add LOG_PATH config key to define a custom log file

README.md

@@ -53,7 +53,7 @@ bulk insert DBNAME/TABLENAME from PATH with (CONFIG)
     TABLENAME is optional and will be derived from the directory name's file if not specified.
 
     (CONFIG) is a configuration string with the following format:
-        (FILE_DEL=1/0,KEYS=N,MZ=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp)
+        (FILE_DEL=1/0,KEYS=N,MZ=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp,LOG_PATH=/path/to/file.log)
 
         Where 1/0 represents "true" / "false", and N is an integer.
         FILE_DEL: Delete file after importation is completed.
@@ -71,6 +71,7 @@ bulk insert DBNAME/TABLENAME from PATH with (CONFIG)
         MAX_RECORD: Maximum record size in bytes (Default value: 2048).
         MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50.
         TMP_PATH: Path to the folder used for temporary files (default: /tmp).
+        LOG_PATH: Path to a custom log file (default: /var/log/scanoss/ldb/DBNAME.log).
 
 bulk insert DBNAME/TABLENAME from PATH
     Imports data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported.

src/import.c

@@ -1232,6 +1232,7 @@ const char * config_parameters[] = {
 									"MAX_RECORD",
 									"MAX_RAM_PERCENT",
 									"TMP_PATH",
+									"LOG_PATH",
 									};
 
 #define CONFIG_PARAMETERS_NUMBER  (sizeof(config_parameters) / sizeof(config_parameters[0]))
@@ -1306,10 +1307,24 @@ bool ldb_importation_config_parse(import_params_t * opt, char * line)
 				*c = 0;
 			else
 			{
-				 c = strrchr(opt->params.tmp_path,')');	
+				 c = strrchr(opt->params.tmp_path,')');
 				 if (c)
 				 	*c = 0;
-			} 
+			}
+		}
+		else if (!strcmp(config_parameters[i], "LOG_PATH"))
+		{
+			strncpy(opt->params.log_path,no_spaces + (param - normalized) + 1, LDB_MAX_PATH);
+			//remove spurius ")" or "," from path TODO: improve
+			char * c = strchr(opt->params.log_path,',');
+			if (c)
+				*c = 0;
+			else
+			{
+				 c = strrchr(opt->params.log_path,')');
+				 if (c)
+				 	*c = 0;
+			}
 		}
 		else if (sscanf(param,"=%d", &val))
 		{
@@ -2150,6 +2165,8 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 		ldb_importation_config_parse(&user_opt, config);
 		if (user_opt.params.verbose > 0)
 			logger_set_level(user_opt.params.verbose);
+		if (*user_opt.params.log_path)
+			logger_set_path(user_opt.params.log_path);
 	}
 
 	if (!ldb_database_exists(job.dbname))

src/import.h

@@ -23,6 +23,7 @@ typedef union import_params {
 		int collate_max_rec;
 		int collate_max_ram_percent;
 		char tmp_path[LDB_MAX_PATH];
+		char log_path[LDB_MAX_PATH];
 	} params;
 	int params_arr[IMPORT_PARAMS_NUMBER];
 } import_params_t;

src/logger.c

@@ -132,13 +132,8 @@ void log_info(const char * fmt, ...)
 	pthread_mutex_unlock(&logger_lock);
 }
 
-void logger_dbname_set(char * db)
+static void logger_write_header(void)
 {
-    if (*import_logger_path)    
-        return;
-
-    ldb_prepare_dir(LOGGER_DIR);
-    sprintf(import_logger_path, "%s/%s.log", LOGGER_DIR, db);
     time_t currentTime = time(NULL);
 	struct tm *localTime = localtime(&currentTime);
 	char timeString[64];
@@ -151,6 +146,25 @@ void logger_dbname_set(char * db)
     }
 }
 
+void logger_set_path(const char * path)
+{
+    if (*import_logger_path || !path || !*path)
+        return;
+
+    strncpy(import_logger_path, path, LDB_MAX_PATH - 1);
+    logger_write_header();
+}
+
+void logger_dbname_set(char * db)
+{
+    if (*import_logger_path)
+        return;
+
+    ldb_prepare_dir(LOGGER_DIR);
+    sprintf(import_logger_path, "%s/%s.log", LOGGER_DIR, db);
+    logger_write_header();
+}
+
 void logger_init(char * db, int tnumber,  pthread_t * tlist)
 {
     pthread_mutex_init(&logger_lock, NULL);

src/logger.h

@@ -17,6 +17,7 @@ void log_info(const char * fmt, ...);
 void logger_set_level(log_level_t l);
 void log_debug(const char * fmt, ...);
 void logger_dbname_set(char * db);
+void logger_set_path(const char * path);
 void logger_basic(const char * fmt, ...);
 #define LOG_INF(fmt,args...) import_logger(NULL, fmt, args)
 #endif

src/shell.c

@@ -71,7 +71,7 @@ void help()
 	printf("Import data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported.\n");
 	printf("TABLENAME is optional and will be defined from the directory name's file if not specified.\n");
 	printf("(CONFIG) is a configuration string with the following format:\n");
-	printf("    (FILE_DEL=1/0,KEYS=N,MZ=1/0,BIN=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp)\n");
+	printf("    (FILE_DEL=1/0,KEYS=N,MZ=1/0,BIN=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp,LOG_PATH=/path/to/file.log)\n");
 	printf("    Where 1/0 represents true/false, and N is an integer.\n");
 	printf("    FILE_DEL: Delete file after importation is complete.\n");
 	printf("    KEYS: Number of binary keys in the CSV file.\n");
@@ -88,6 +88,7 @@ void help()
 	printf("    MAX_RECORD: define the max record size, if a sector is bigger than \"MAX_RECORD\" bytes will be removed.\n");
 	printf("    MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50.\n");
 	printf("    TMP_PATH: Define the temporary directory. Default value \"/tmp\".\n");
+	printf("    LOG_PATH: Define a custom log file. Default value \"/var/log/scanoss/ldb/DBNAME.log\".\n");
 	printf("	It is not mandatory to specify all parameters; default values will be assumed for missing parameters.\n\n");
 
 	printf("	bulk insert  DBNAME/TABLENAME from PATH\n");

test/test_kb

@@ -102,6 +102,27 @@ test_10_configuration() {
     assert_equals "$config" "$result"
 }
 
+test_11_log_path() {
+    custom_log="/tmp/test_kb_custom.log"
+    rm -f $custom_log
+    default_log_lines=$(wc -l < /var/log/scanoss/ldb/test_kb.log)
+
+    echo "bulk insert test_kb/file from source/mined/file with (TMP_PATH=/var,FILE_DEL=0,LOG_PATH=$custom_log)" | ../ldb -q
+    assert "test -e $custom_log" "custom log file was not created"
+
+    result=$(head -1 $custom_log)
+    assert_equals "Exec Time:" "$(echo $result | cut -d' ' -f1-2)" "custom log header fails"
+
+    cores=$(nproc)
+    threads=$((cores / 2))
+    config="file configuration: (KEYS=2, VALIDATE_FIELDS=1, FIELDS=3, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=$threads, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/var)"
+    result=$(tac $custom_log | grep -m 1 "file configuration:")
+    assert_equals "$config" "$result" "custom log content fails"
+
+    assert_equals "$default_log_lines" "$(wc -l < /var/log/scanoss/ldb/test_kb.log)" "default log was written despite LOG_PATH"
+    rm -f $custom_log
+}
+
 setup_suite () {
     ../ldb -u source/mined -n test_kb
 }