Adding a command-line interface, CSV field resolver, and usage examples

README.md

@@ -49,10 +49,27 @@ conf.setNaaccrVersion("220");
 new MetafileTranslator().executeFullTranslation(conf);
 ```
 
+## Command Line
+The compiled JAR can be executed from the command line to perform a full translation.
+
+**Command Line Options:**
+- `-p`: The working folder containing the metafile
+- `-c`: The file name of the configuration properties file within the working folder
+
+**Example usage:**
+```sh
+$ java -jar validation-translation.jar -c config.properties -p WORKING_DIR
+```
+For an example of a translation configuration, see `examples/config.properties`
+
+## Pre-Compiled Groovy Edits
+For faster loading of large metafiles, the optionally generated Groovy source code can be pre-compiled into bytecode. The folder `examples/compile-groovy` 
+contains an example of a build.gradle and settings.gradle that can be placed in the output folder to generate a JAR containing compiled bytecode.
+
 ## About SEER
 
 This library was developed through the [SEER](http://seer.cancer.gov/) program.
 
 The Surveillance, Epidemiology and End Results program is a premier source for cancer statistics in the United States.
 The SEER program collects information on incidence, prevalence and survival from specific geographic areas representing
-a large portion of the US population and reports on all these data plus cancer mortality data for the entire country.
+a large portion of the US population and reports on all these data plus cancer mortality data for the entire country.

build.gradle

@@ -31,10 +31,12 @@ dependencies {
     implementation 'net.sf.squirrel-sql.thirdparty-non-maven:java-cup:0.11a'
     implementation 'org.xerial:sqlite-jdbc:3.43.2.1'
     implementation 'org.apache.logging.log4j:log4j-api:2.21.1'
+    implementation 'commons-cli:commons-cli:1.6.0'
+    implementation 'org.apache.logging.log4j:log4j-core:2.21.1'
+    implementation 'org.slf4j:slf4j-nop:2.0.12'
 
     testImplementation 'junit:junit:4.13.2'
     testImplementation 'com.imsweb:data-generator:1.31'
-    testImplementation 'org.apache.logging.log4j:log4j-core:2.21.1'
 }
 
 // enforce UTF-8, display the compilation warnings
@@ -67,7 +69,7 @@ jar {
                 'Built-By': System.getProperty('user.name'),
                 'Built-Date': new Date(),
                 'Built-JDK': System.getProperty('java.version'),
-                'Automatic-Module-Name': 'com.imsweb.validation-translation'
+                'Automatic-Module-Name': 'com.imsweb.validation-translation',
         )
     }
 }
@@ -140,11 +142,34 @@ tasks.register('generateRegexParser') {
     }
 }
 
+// use this task to generate a fat JAR to easily run a CLI
+tasks.register('buildFatJar',Jar) {
+    manifest {
+        attributes('Implementation-Title': project.name,
+                'Implementation-Version': project.version,
+                'Implementation-Vendor': 'Information Management Services Inc.',
+                'Created-By': System.properties.getProperty('java.vm.version') + ' (' + System.properties.getProperty('java.vm.vendor') + ')',
+                'Built-By': System.getProperty('user.name'),
+                'Built-Date': new Date(),
+                'Built-JDK': System.getProperty('java.version'),
+                'Automatic-Module-Name': 'com.imsweb.validation-translation',
+                'Main-Class': 'com.imsweb.validation.translation.MetafileTranslatorCli'
+        )
+    }
+
+    from {
+        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
+    }
+    duplicatesStrategy=DuplicatesStrategy.EXCLUDE
+    with jar
+}
+
 // Nexus vulnerability scan (https://github.com/sonatype-nexus-community/scan-gradle-plugin)
 ossIndexAudit {
     outputFormat = 'DEPENDENCY_GRAPH'
     printBanner = false
 
+
     excludeVulnerabilityIds = [
             'CVE-2022-42003',
             'CVE-2022-42004',

examples/config.properties

@@ -0,0 +1,16 @@
+# metafile filename. should be located in the working path along with this configuration file
+metafile-name=testing-metafile-v2.smf
+# prefix for the validator rule IDs to generate
+translation-prefix=TEST
+# working directory for a previous translation run (optional). this will be relative to the path where the program is executed
+previous-working-directory-path=testing-metafile-v1
+# base naaccr dictionary version to use to resolve fields
+naaccr-version=230
+# user dictionary for resolving fields should the base naaccr dictionary fail to resolve a field
+user-dictionary-file=test-user-dictionary.xml
+# field mapping csv file for resolving fields should the base and user naaccr dictionaries fail to resolve a field
+field-mappings-file=test-field-mappings.csv
+# generate groovy source file? 1, yes, or true will enable groovy source generation
+generate-groovy-src=1
+# number of files to split the groovy source into. this can help if groovy compilation runs out of memory.
+groovy-src-num-files=2

examples/test-field-mappings.csv

@@ -0,0 +1,5 @@
+1234,xmlFieldNameMappingForNaaccrId
+9101,stateSpecificField1
+9102,stateSpecificField2
+10010,userDictionaryField1
+10020,userDictionaryField2

src/main/java/com/imsweb/validation/translation/CsvFieldResolver.java

@@ -0,0 +1,84 @@
+package com.imsweb.validation.translation;
+
+import com.imsweb.naaccrxml.NaaccrXmlDictionaryUtils;
+import com.imsweb.validation.translation.metafile.MetafileField;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Implementation of the FieldResolver that reads a comma-delimted file containing item number
+ * and field name and uses that to supplement NAACCR and user dictionary fields when resolving
+ * fields used in a EDITS metafile.
+ * (note: does not conform to RFC 4180! quoting and escaping are not supported)
+ */
+@SuppressWarnings("unused")
+public class CsvFieldResolver extends FieldResolver {
+    private static final Logger _LOG = LogManager.getLogger(CsvFieldResolver.class);
+
+    private Map<Integer, String> _mappings;
+
+    public CsvFieldResolver() {
+        setMappings(new HashMap<>());
+    }
+
+    /**
+     * Reads the comma delimited file into memory. If mappings are already in place,
+     * new mappings are loaded over top of the old ones. This way, multiple mapping files
+     * can be loaded.
+     * @param mappingsFile
+     * @throws IOException
+     */
+    public void loadMappingsFile(File mappingsFile) throws IOException {
+        if (mappingsFile == null) {
+            throw new IOException("Mappings file cannot be null");
+        }
+        if (! mappingsFile.exists()) {
+            throw new IOException("Mappings file does not exist");
+        }
+
+        for (String line : Files.readAllLines(mappingsFile.toPath(), StandardCharsets.UTF_8)) {
+            String[] mapping = line.trim().split("\\s*,\\s*");
+            if (mapping.length != 2) {
+               throw new IOException("Invalid line in mapping file: " + line);
+            }
+            getMappings().put(Integer.parseInt(mapping[0]), mapping[1]);
+            _LOG.info("      >> Loaded mapping from " + mappingsFile.getName() + ": " + mapping[0] + "->" + mapping[1]);
+        }
+    }
+
+    @Override
+    protected String resolveFieldPostDictionary(MetafileField field, TranslationConfiguration conf) {
+        String propName = super.resolveFieldPostDictionary(field, conf);
+
+        // if unable to resolve by superclass implementation, try the in memory mapping
+        if (propName == null) {
+            int itemNumber = field.getNumber();
+            if (getMappings().containsKey(itemNumber)) {
+                propName = getMappings().get(itemNumber);
+            }
+        }
+
+        // if unable to resolve from the mapping file, create a (properly named) dummy item and warn the user
+        if (propName == null) {
+            propName = NaaccrXmlDictionaryUtils.createNaaccrIdFromItemName(field.getName());
+            _LOG.info("      >> Unsupported field: " + field.getName() + " (#" + field.getNumber() + "); deriving the ID from the name: " + propName);
+        }
+        return propName;
+    }
+
+    public Map<Integer, String> getMappings() {
+        return _mappings;
+    }
+
+    public void setMappings(Map<Integer, String> mappings) {
+        this._mappings = mappings;
+    }
+
+}

src/main/java/com/imsweb/validation/translation/MetafileTranslatorCli.java

@@ -0,0 +1,162 @@
+package com.imsweb.validation.translation;
+
+import com.imsweb.naaccrxml.NaaccrXmlDictionaryUtils;
+import com.imsweb.naaccrxml.entity.dictionary.NaaccrDictionary;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+public class MetafileTranslatorCli {
+    private static final Logger _LOG = LogManager.getLogger(MetafileTranslatorCli.class);
+
+    public static void main(String[] args) 
+        throws Exception {
+        CommandLineParser parser = new DefaultParser();
+
+
+        // set up options
+        Options opts = new Options();
+        opts.addRequiredOption("p", "path", true, "Path to working directory containing metafile where output will be placed");
+        opts.addRequiredOption("c", "config", true, "Filename of configuration properties file within the working directory to use for this translation");
+        opts.addOption("h", "help", false, "Prints this help message");
+
+        try {
+            CommandLine cmdLine = parser.parse(opts, args);
+
+            // if help requested, print and exit
+            if (cmdLine.hasOption("h")) {
+                HelpFormatter helpFormatter = new HelpFormatter();
+                helpFormatter.printHelp(MetafileTranslatorCli.class.getName(), opts);
+                return;
+            }
+
+            // process options into translation configuration
+            TranslationConfiguration conf = getConfiguration(cmdLine.getOptionValue("p"), cmdLine.getOptionValue("c"));
+
+            // run the translation
+            new MetafileTranslator().executeFullTranslation(conf);
+        } 
+        catch (ParseException e) {
+            HelpFormatter helpFormatter = new HelpFormatter();
+            helpFormatter.printHelp("validation-translation", opts);
+        }
+    }
+
+    /**
+     * Creates a TranslationConfiguration object from the configuration properties file
+     * @param path - path to the directory containing the configuration properties file
+     * @param filename - name of the configuration properties file
+     * @return
+     */
+    private static TranslationConfiguration getConfiguration(String path, String filename)
+            throws IOException, TranslationException {
+        // make sure the directory exists
+        File configDir = new File(path);
+        if (! configDir.exists() || ! configDir.isDirectory()) {
+            throw new IOException("Working directory path must point to an existing directory");
+        }
+
+        // check to see if the properties file exists
+        File configFile = new File(configDir, filename);
+        if (! configFile.exists() || ! configFile.isFile()) {
+            throw new IOException("Configuration file not found");
+        }
+
+        // load the properties file
+        Properties configProperties = new Properties();
+        try (InputStream inputStream = Files.newInputStream(configFile.toPath())) {
+            configProperties.load(inputStream);
+        }
+
+        // set up the translation configuration based on contents of the config properties file
+        TranslationConfiguration translationConfiguration = new TranslationConfiguration();
+        translationConfiguration.setWorkingDirectoryPath(path);
+
+        // load the required items
+        if (configProperties.containsKey("metafile-name")) {
+            translationConfiguration.setMetafileName(configProperties.getProperty("metafile-name"));
+        }
+        else {
+            throw new TranslationException("No metafile name specified in configuration");
+        }
+
+        if (configProperties.containsKey("translation-prefix")) {
+            translationConfiguration.setTranslationPrefix(configProperties.getProperty("translation-prefix"));
+        }
+        else {
+            throw new TranslationException("Translation prefix is not specified in configuration");
+        }
+
+        if (configProperties.containsKey("naaccr-version")) {
+            translationConfiguration.setNaaccrVersion(configProperties.getProperty("naaccr-version"));
+        }
+        else {
+            throw new TranslationException("No NAACCR version specified in configuration");
+        }
+
+        // load optional previous translation output path
+        if (configProperties.containsKey("previous-working-directory-path")) {
+            File previousWorkingDirectoryPath = new File(configProperties.getProperty("previous-working-directory-path"));
+            if (! previousWorkingDirectoryPath.exists() || !previousWorkingDirectoryPath.isDirectory()) {
+                throw new IOException("Previous working directory specified in configuration file not found");
+            }
+            translationConfiguration.setPreviousWorkingDirectoryPath(configProperties.getProperty("previous-working-directory-path"));
+        }
+
+        // if a field mapping csv file is specified then create a new CsvFieldResolver to handle it
+        if (configProperties.containsKey("field-mappings-file")) {
+            File fieldMappingsFile = new File(configDir, configProperties.getProperty("field-mappings-file"));
+            if (! fieldMappingsFile.exists() || ! fieldMappingsFile.isFile()) {
+                throw new IOException("Field mappings file specified in configuration file not found");
+            }
+            CsvFieldResolver csvFieldResolver = new CsvFieldResolver();
+            csvFieldResolver.loadMappingsFile(fieldMappingsFile);
+
+            translationConfiguration.setFieldResolver(csvFieldResolver);
+        }
+
+        // if a user dictionary is specified, load it
+        if (configProperties.containsKey("user-dictionary-file")) {
+            File userDictionaryFile = new File(configDir, configProperties.getProperty("user-dictionary-file"));
+            if (! userDictionaryFile.exists() || ! userDictionaryFile.isFile()) {
+                throw new IOException("User dictionary file specified in configuration file not found");
+            }
+
+            List<NaaccrDictionary> userDictionaries = new ArrayList<NaaccrDictionary>(1);
+            NaaccrDictionary dict = NaaccrXmlDictionaryUtils.readDictionary(userDictionaryFile);
+            userDictionaries.add(dict);
+            _LOG.info("      >> Loaded user dictionary " + userDictionaryFile.getName());
+
+            translationConfiguration.setUserDefinedDictionaries(userDictionaries);
+        }
+
+        // check to see if groovy source code needs to be generated
+        if (configProperties.containsKey("generate-groovy-src")) {
+            // if it is truthy, enable source generation
+            if (configProperties.getProperty("generate-groovy-src").equals("1")
+                    || configProperties.getProperty("generate-groovy-src").equalsIgnoreCase("yes")
+                    || configProperties.getProperty("generate-groovy-src").equalsIgnoreCase("true")) {
+                translationConfiguration.setGenerateGroovySourceCode(true);
+            }
+        }
+        // check if groovy source needs to be split
+        if (configProperties.containsKey("groovy-src-num-files")) {
+            translationConfiguration.setGroovySourceCodeNumFiles(Integer.parseInt(configProperties.getProperty("groovy-src-num-files")));
+        }
+
+        return translationConfiguration;
+    }
+}

src/main/resources/log4j2.properties

@@ -0,0 +1,5 @@
+rootLogger=INFO, STDOUT
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n