diff --git a/README.md b/README.md index f99e478..39deff8 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ bulk insert DBNAME/TABLENAME from PATH with (CONFIG) TABLENAME is optional and will be derived from the directory name's file if not specified. (CONFIG) is a configuration string with the following format: - (FILE_DEL=1/0,KEYS=N,MZ=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp) + (FILE_DEL=1/0,KEYS=N,MZ=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp,LOG_PATH=/path/to/file.log) Where 1/0 represents "true" / "false", and N is an integer. FILE_DEL: Delete file after importation is completed. @@ -71,6 +71,7 @@ bulk insert DBNAME/TABLENAME from PATH with (CONFIG) MAX_RECORD: Maximum record size in bytes (Default value: 2048). MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50. TMP_PATH: Path to the folder used for temporary files (default: /tmp). + LOG_PATH: Path to a custom log file (default: /var/log/scanoss/ldb/DBNAME.log). bulk insert DBNAME/TABLENAME from PATH Imports data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported. diff --git a/src/import.c b/src/import.c index ae26f24..2c217fa 100644 --- a/src/import.c +++ b/src/import.c @@ -1232,6 +1232,7 @@ const char * config_parameters[] = { "MAX_RECORD", "MAX_RAM_PERCENT", "TMP_PATH", + "LOG_PATH", }; #define CONFIG_PARAMETERS_NUMBER (sizeof(config_parameters) / sizeof(config_parameters[0])) @@ -1306,10 +1307,24 @@ bool ldb_importation_config_parse(import_params_t * opt, char * line) *c = 0; else { - c = strrchr(opt->params.tmp_path,')'); + c = strrchr(opt->params.tmp_path,')'); if (c) *c = 0; - } + } + } + else if (!strcmp(config_parameters[i], "LOG_PATH")) + { + strncpy(opt->params.log_path,no_spaces + (param - normalized) + 1, LDB_MAX_PATH); + //remove spurius ")" or "," from path TODO: improve + char * c = strchr(opt->params.log_path,','); + if (c) + *c = 0; + else + { + c = strrchr(opt->params.log_path,')'); + if (c) + *c = 0; + } } else if (sscanf(param,"=%d", &val)) { @@ -2150,6 +2165,8 @@ bool ldb_import_command(char * dbtable, char * path, char * config) ldb_importation_config_parse(&user_opt, config); if (user_opt.params.verbose > 0) logger_set_level(user_opt.params.verbose); + if (*user_opt.params.log_path) + logger_set_path(user_opt.params.log_path); } if (!ldb_database_exists(job.dbname)) diff --git a/src/import.h b/src/import.h index 17251ed..bb04039 100644 --- a/src/import.h +++ b/src/import.h @@ -23,6 +23,7 @@ typedef union import_params { int collate_max_rec; int collate_max_ram_percent; char tmp_path[LDB_MAX_PATH]; + char log_path[LDB_MAX_PATH]; } params; int params_arr[IMPORT_PARAMS_NUMBER]; } import_params_t; diff --git a/src/logger.c b/src/logger.c index 2ca4532..24f3a06 100644 --- a/src/logger.c +++ b/src/logger.c @@ -132,13 +132,8 @@ void log_info(const char * fmt, ...) pthread_mutex_unlock(&logger_lock); } -void logger_dbname_set(char * db) +static void logger_write_header(void) { - if (*import_logger_path) - return; - - ldb_prepare_dir(LOGGER_DIR); - sprintf(import_logger_path, "%s/%s.log", LOGGER_DIR, db); time_t currentTime = time(NULL); struct tm *localTime = localtime(¤tTime); char timeString[64]; @@ -151,6 +146,25 @@ void logger_dbname_set(char * db) } } +void logger_set_path(const char * path) +{ + if (*import_logger_path || !path || !*path) + return; + + strncpy(import_logger_path, path, LDB_MAX_PATH - 1); + logger_write_header(); +} + +void logger_dbname_set(char * db) +{ + if (*import_logger_path) + return; + + ldb_prepare_dir(LOGGER_DIR); + sprintf(import_logger_path, "%s/%s.log", LOGGER_DIR, db); + logger_write_header(); +} + void logger_init(char * db, int tnumber, pthread_t * tlist) { pthread_mutex_init(&logger_lock, NULL); diff --git a/src/logger.h b/src/logger.h index b4648ef..0a59987 100644 --- a/src/logger.h +++ b/src/logger.h @@ -17,6 +17,7 @@ void log_info(const char * fmt, ...); void logger_set_level(log_level_t l); void log_debug(const char * fmt, ...); void logger_dbname_set(char * db); +void logger_set_path(const char * path); void logger_basic(const char * fmt, ...); #define LOG_INF(fmt,args...) import_logger(NULL, fmt, args) #endif \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index b55f2bd..b80e8a9 100644 --- a/src/shell.c +++ b/src/shell.c @@ -71,7 +71,7 @@ void help() printf("Import data from PATH into the specified db/table. If PATH is a directory, its files will be recursively imported.\n"); printf("TABLENAME is optional and will be defined from the directory name's file if not specified.\n"); printf("(CONFIG) is a configuration string with the following format:\n"); - printf(" (FILE_DEL=1/0,KEYS=N,MZ=1/0,BIN=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp)\n"); + printf(" (FILE_DEL=1/0,KEYS=N,MZ=1/0,BIN=1/0,WFP=1/0,OVERWRITE=1/0,SORT=1/0,FIELDS=N,VALIDATE_FIELDS=1/0,VALIDATE_VERSION=1/0,VERBOSE=1/0,COLLATE=1/0,MAX_RECORD=N,TMP_PATH=/path/to/tmp,LOG_PATH=/path/to/file.log)\n"); printf(" Where 1/0 represents true/false, and N is an integer.\n"); printf(" FILE_DEL: Delete file after importation is complete.\n"); printf(" KEYS: Number of binary keys in the CSV file.\n"); @@ -88,6 +88,7 @@ void help() printf(" MAX_RECORD: define the max record size, if a sector is bigger than \"MAX_RECORD\" bytes will be removed.\n"); printf(" MAX_RAM_PERCENT: limit the system RAM usage during collate process. Default value: 50.\n"); printf(" TMP_PATH: Define the temporary directory. Default value \"/tmp\".\n"); + printf(" LOG_PATH: Define a custom log file. Default value \"/var/log/scanoss/ldb/DBNAME.log\".\n"); printf(" It is not mandatory to specify all parameters; default values will be assumed for missing parameters.\n\n"); printf(" bulk insert DBNAME/TABLENAME from PATH\n"); diff --git a/test/test_kb b/test/test_kb index 5700fc8..e61b3af 100644 --- a/test/test_kb +++ b/test/test_kb @@ -102,6 +102,27 @@ test_10_configuration() { assert_equals "$config" "$result" } +test_11_log_path() { + custom_log="/tmp/test_kb_custom.log" + rm -f $custom_log + default_log_lines=$(wc -l < /var/log/scanoss/ldb/test_kb.log) + + echo "bulk insert test_kb/file from source/mined/file with (TMP_PATH=/var,FILE_DEL=0,LOG_PATH=$custom_log)" | ../ldb -q + assert "test -e $custom_log" "custom log file was not created" + + result=$(head -1 $custom_log) + assert_equals "Exec Time:" "$(echo $result | cut -d' ' -f1-2)" "custom log header fails" + + cores=$(nproc) + threads=$((cores / 2)) + config="file configuration: (KEYS=2, VALIDATE_FIELDS=1, FIELDS=3, VALIDATE_VERSION=1, SORT=1, FILE_DEL=0, OVERWRITE=0, WFP=0, MZ=0, VERBOSE=0, THREADS=$threads, COLLATE=0, MAX_RECORD=2048, MAX_RAM_PERCENT=50, TMP_PATH=/var)" + result=$(tac $custom_log | grep -m 1 "file configuration:") + assert_equals "$config" "$result" "custom log content fails" + + assert_equals "$default_log_lines" "$(wc -l < /var/log/scanoss/ldb/test_kb.log)" "default log was written despite LOG_PATH" + rm -f $custom_log +} + setup_suite () { ../ldb -u source/mined -n test_kb }