Parse better templates

aom/src/main/java/com/nedap/archie/aom/primitives/CTerminologyCode.java

@@ -173,9 +173,9 @@ public boolean cConformsTo(CObject other, BiFunction<String, String, Boolean> rm
             }
             for (String value : valueSet) {
                 //TODO: redefine validation to actually work here!
-//                if (!otherValueSet.contains(value)) {
-//                    return false;
-//                }
+                if (!otherValueSet.contains(value)) {
+                    return false;
+                }
             }
             return true;
         } else {

better-template/build.gradle

@@ -0,0 +1,15 @@
+description = "A parser and converter for the Better Systems JSON template format"
+
+dependencies {
+  compile project(':grammars')
+  compile project(':base')
+  compile project(':aom')
+  compile project(':bmm') 
+  compile project(':path-queries')
+  compile project(':utils')
+  compile project(':tools')
+  compile project(':openehr-rm')
+  compile project(':archie-utils')
+  compile project(':i18n')
+  compile project(':referencemodels')
+}

better-template/src/main/java/com/nedap/archie/template/betterjson/ArchetypeTermFixer.java

@@ -0,0 +1,196 @@
+package com.nedap.archie.template.betterjson;
+
+import com.nedap.archie.aom.Archetype;
+import com.nedap.archie.aom.CArchetypeRoot;
+import com.nedap.archie.aom.CAttribute;
+import com.nedap.archie.aom.CComplexObject;
+import com.nedap.archie.aom.CObject;
+import com.nedap.archie.aom.CPrimitiveObject;
+import com.nedap.archie.aom.Template;
+import com.nedap.archie.aom.TemplateOverlay;
+import com.nedap.archie.aom.terminology.ArchetypeTerm;
+import com.nedap.archie.aom.terminology.ArchetypeTerminology;
+import com.nedap.archie.aom.terminology.ValueSet;
+import com.nedap.archie.aom.utils.AOMUtils;
+import com.nedap.archie.aom.utils.NodeIdUtil;
+import com.nedap.archie.base.Interval;
+import com.nedap.archie.flattener.InMemoryFullArchetypeRepository;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class ArchetypeTermFixer {
+
+    private String originalLanguage;
+
+    public void fixTerms(Archetype archetype, FlatArchetypeProvider repo) {
+        originalLanguage = archetype.getOriginalLanguage().getCodeString();
+        addTerminologyIfNotPresent(archetype);
+        fixTerms(archetype,repo, archetype.getDefinition());
+        fixValueSetCodes(archetype, repo);
+
+        removeUntranslatedLanguages(archetype);
+
+        if(archetype instanceof Template) {
+            Template template = (Template) archetype;
+            for(TemplateOverlay overlay:template.getTemplateOverlays()) {
+                addTerminologyIfNotPresent(overlay);
+                fixTerms(overlay, repo, overlay.getDefinition());
+                fixValueSetCodes(overlay, repo);
+            }
+        }
+    }
+
+    private void fixValueSetCodes(Archetype archetype, FlatArchetypeProvider repo) {
+        String language = originalLanguage;
+        if(!archetype.getTerminology().getTermDefinitions().containsKey(originalLanguage)) {
+            language = archetype.getTerminology().getTermDefinitions().keySet().iterator().next();
+        }
+        if(archetype.getTerminology() != null && archetype.getTerminology().getValueSets() != null) {
+            for(String code:archetype.getTerminology().getValueSets().keySet()) {
+                if(!archetype.getTerminology().getTermDefinitions().get(language).containsKey(code) &&
+                        AOMUtils.getSpecializationDepthFromCode(code) == archetype.specializationDepth()) {
+                    Archetype flatParent = repo.getFlatArchetype(archetype.getParentArchetypeId());
+                    createTermForNewCodeWithFlatParent(archetype, code, flatParent);
+                }
+                for(String valueCode:archetype.getTerminology().getValueSets().get(code).getMembers()) {
+                    if(!archetype.getTerminology().getTermDefinitions().get(language).containsKey(valueCode) &&
+                            AOMUtils.getSpecializationDepthFromCode(valueCode) == archetype.specializationDepth()) {
+                        Archetype flatParent = repo.getFlatArchetype(archetype.getParentArchetypeId());
+                        createTermForNewCodeWithFlatParent(archetype, valueCode, flatParent);
+                    }
+                }
+            }
+        }
+    }
+
+    private void addTerminologyIfNotPresent(Archetype archetype) {
+        if(archetype.getTerminology() == null) {
+            archetype.setTerminology(new ArchetypeTerminology());
+        }
+        if(archetype.getTerminology().getTermDefinitions().isEmpty()) {
+            archetype.getTerminology().getTermDefinitions().put(originalLanguage, new LinkedHashMap<>());
+        }
+    }
+
+    /** If any languages do not exist in the terminology, remove from translations*/
+    private void removeUntranslatedLanguages(Archetype archetype) {
+        if(archetype.getTranslations() == null) {
+            return;
+        }
+        List<String> toRemove = new ArrayList<>();
+        for(String translation:archetype.getTranslations().keySet()) {
+            if(archetype.getTerminology() != null && !archetype.getTerminology().getTermDefinitions().containsKey(translation)) {
+                toRemove.add(translation);
+            }
+        }
+        if(!toRemove.isEmpty()) {
+            for(String translation:toRemove) {
+                archetype.getTranslations().remove(translation);
+            }
+        }
+    }
+
+    private void fixTerms(Archetype archetype, FlatArchetypeProvider repo, CComplexObject cObject) {
+        String language = originalLanguage;
+        if(!archetype.getTerminology().getTermDefinitions().containsKey(originalLanguage)) {
+            language = archetype.getTerminology().getTermDefinitions().keySet().iterator().next();
+        }
+        if(cObject instanceof CArchetypeRoot) {
+            if(!archetype.getTerminology().getTermDefinitions().get(language).containsKey(cObject.getNodeId()) &&
+                    archetype.specializationDepth() == AOMUtils.getSpecializationDepthFromCode(cObject.getNodeId())
+            ) {
+                Archetype referencedArchetype = repo.getFlatArchetype(((CArchetypeRoot) cObject).getArchetypeRef());
+                createTermForNewCodeWithRoot(archetype, cObject.getNodeId(), referencedArchetype);
+                //TODO: fix lots of problems where node ids are set wrong, for example id2.1 to set the occurrences of id2 to {0} is a problem!
+            }
+        } else if(cObject instanceof CComplexObject) {
+
+            if(!archetype.getTerminology().getTermDefinitions().get(language).containsKey(cObject.getNodeId()) &&
+                    archetype.specializationDepth() == AOMUtils.getSpecializationDepthFromCode(cObject.getNodeId())
+            ) {
+                Archetype flatParent = repo.getFlatArchetype(archetype.getParentArchetypeId());
+                createTermForNewCodeWithFlatParent(archetype, cObject.getNodeId(), flatParent);
+                //TODO: fix lots of problems where node ids are set wrong, for example id2.1 to set the occurrences of id2 to {0} is a problem!
+            }
+        }
+        for(CAttribute attribute:cObject.getAttributes()) {
+            fixTerms(archetype, repo, attribute);
+        }
+    }
+
+    private void fixTerms(Archetype archetype, FlatArchetypeProvider repo, CAttribute cAttribute) {
+
+        for(CObject cObject:cAttribute.getChildren()) {
+            if(cObject instanceof CComplexObject) {
+                fixTerms(archetype, repo, (CComplexObject) cObject);
+            }
+        }
+    }
+
+    private static Pattern synthesizedCodesPattern = Pattern.compile("(id)(0\\.)*9[0-9][0-9][0-9](\\.[0-9]*)*");
+
+    private void createTermForNewCodeWithRoot(Archetype archetype, String code, Archetype referencedArchetype) {
+        if(!synthesizedCodesPattern.matcher(code).matches()) {
+            //if(cObject.getParent().isMultiple()) {
+            for (String language : archetype.getTerminology().getTermDefinitions().keySet()) {
+                //TODO: add new archetype term to conversion log?
+
+                ArchetypeTerm newTerm = new ArchetypeTerm();
+                newTerm.setCode(code);
+                ArchetypeTerm rootTerm = null;
+                if (referencedArchetype != null) {
+                    rootTerm = referencedArchetype.getTerm(referencedArchetype.getDefinition(), language);
+                    if(rootTerm == null) {
+                       rootTerm = referencedArchetype.getDefinition().getTerm();
+                    }
+                    if(rootTerm == null) {
+                        //yeas this is persistent
+                        rootTerm = referencedArchetype.getTerminology().getTermDefinition("en", "id1");
+                    }
+                }
+
+                newTerm.setText(rootTerm == null ? "* missing code" : rootTerm.getText());
+                newTerm.setDescription(rootTerm == null ? "* missing code" : rootTerm.getDescription());
+                archetype.getTerminology().getTermDefinitions().get(language).put(newTerm.getCode(), newTerm);
+
+            }
+        }
+       // }
+    }
+
+    private void createTermForNewCodeWithFlatParent(Archetype archetype, String code, Archetype flatParent) {
+        if(!synthesizedCodesPattern.matcher(code).matches()) {
+            //TODO: better would be, but difficult to do correctly:
+            // if(cObject.getParent().isMultiple()) {
+            for (String language : archetype.getTerminology().getTermDefinitions().keySet()) {
+                //TODO: add new archetype term to conversion log?
+
+                ArchetypeTerm newTerm = new ArchetypeTerm();
+                newTerm.setCode(code);
+                ArchetypeTerm parentTerm = null;
+                if (flatParent != null) {
+                    ArchetypeTerminology parentTerminology = flatParent.getTerminology();
+                    if(parentTerminology != null) {
+                        if(parentTerminology.getTermDefinitions().get(language) != null) {
+                            parentTerm = parentTerminology.getTermDefinition(language, AOMUtils.codeAtLevel(code, flatParent.specializationDepth()));
+                        } else {
+                            parentTerm = parentTerminology.getTermDefinition(flatParent.getOriginalLanguage().getCodeString(), AOMUtils.codeAtLevel(code, flatParent.specializationDepth()));
+                        }
+                    }
+                }
+
+                newTerm.setText(parentTerm == null ? "* missing code" : parentTerm.getText());
+                newTerm.setDescription(parentTerm == null ? "* missing code" : parentTerm.getDescription());
+                archetype.getTerminology().getTermDefinitions().get(language).put(newTerm.getCode(), newTerm);
+
+            }
+        }
+        // }
+    }
+
+
+
+}

better-template/src/main/java/com/nedap/archie/template/betterjson/ExternalTermBindingTranslationFixer.java

@@ -0,0 +1,95 @@
+package com.nedap.archie.template.betterjson;
+
+import com.nedap.archie.adl14.ADL14ConversionConfiguration;
+import com.nedap.archie.adl14.ADL14ConversionUtil;
+import com.nedap.archie.aom.Archetype;
+import com.nedap.archie.aom.CAttribute;
+import com.nedap.archie.aom.CObject;
+import com.nedap.archie.aom.Template;
+import com.nedap.archie.aom.TemplateOverlay;
+import com.nedap.archie.aom.terminology.ArchetypeTerm;
+import com.nedap.archie.base.terminology.TerminologyCode;
+import com.nedap.archie.template.betterjson.parser.TemplateCTerminologyCode;
+import com.nedap.archie.template.betterjson.parser.TemplateTermCode;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+public class ExternalTermBindingTranslationFixer {
+
+    public void fixTranslations(ADL14ConversionConfiguration config, Archetype archetype) {
+        fixInner(config, archetype);
+        if(archetype instanceof Template) {
+            Template template = (Template) archetype;
+            for(TemplateOverlay overlay:template.getTemplateOverlays()) {
+                fixInner(config, overlay);
+            }
+        }
+    }
+
+    private void fixInner(ADL14ConversionConfiguration config, Archetype archetype) {
+        fixInner(config, archetype, archetype.getDefinition());
+    }
+
+    private void fixInner(ADL14ConversionConfiguration config, Archetype archetype, CObject cObject) {
+
+        if(cObject instanceof TemplateCTerminologyCode) {
+            TemplateCTerminologyCode templateCode = (TemplateCTerminologyCode) cObject;
+            if(templateCode.getIncludedExternalTerminologyCodes() != null) {
+                for (TemplateTermCode templateTermCode : templateCode.getIncludedExternalTerminologyCodes()) {
+                    if(templateTermCode.getValue() == null || templateTermCode.getCode() == null || templateTermCode.getTerminologyId() == null) {
+                        System.out.println("STRANGE TERM CODE FOUND");
+                    }
+                    try {
+                        String value = templateTermCode.getValue();
+                        Map<String, URI> uris = archetype.getTerminology().getTermBindings().get(templateTermCode.getTerminologyId());
+                        URI uri = new ADL14ConversionUtil(config).convertToUri(TerminologyCode.createFromString(templateTermCode.getTerminologyId(), null, templateTermCode.getCode()));
+                        String code = findTermbindingCode(uris, uri);
+                        if(code != null) {
+                            //add translation to terminology IF it was auto-generated
+                            for(String language: archetype.getTerminology().getTermDefinitions().keySet()) {
+                                Map<String, ArchetypeTerm> terms = archetype.getTerminology().getTermDefinitions().get(language);
+                                ArchetypeTerm term = terms.get(code);
+                                if(term != null && term.getText().contains("translation not known")) {
+                                    term.setText(value);
+                                    term.setDescription(value);
+                                } else {
+                                    term = new ArchetypeTerm();
+                                    term.setCode(code);
+                                    term.setText(value);
+                                    term.setDescription(value);
+                                    terms.put(code, term);
+                                }
+                            }
+                        }
+                    } catch (URISyntaxException e) {
+                        //not a big problem, just a term will be missing from the terminology.
+                        e.printStackTrace();
+                    }
+
+                }
+            }
+        }
+        for(CAttribute attribute:cObject.getAttributes()) {
+            fixInner(config, archetype, attribute);
+        }
+
+    }
+
+    private String findTermbindingCode(Map<String, URI> uris, URI uri) {
+        for(Map.Entry<String, URI> u:uris.entrySet()) {
+            if(u.getValue().equals(uri)) {
+                return u.getKey();
+            }
+        }
+        return null;
+    }
+
+    private void fixInner(ADL14ConversionConfiguration config, Archetype archetype, CAttribute attribute) {
+
+        for(CObject child:attribute.getChildren()) {
+            fixInner(config, archetype, child);
+        }
+    }
+}

better-template/src/main/java/com/nedap/archie/template/betterjson/FlatArchetypeProvider.java

@@ -0,0 +1,8 @@
+package com.nedap.archie.template.betterjson;
+
+import com.nedap.archie.aom.Archetype;
+
+public interface FlatArchetypeProvider {
+
+    Archetype getFlatArchetype(String archetypeId);
+}

better-template/src/main/java/com/nedap/archie/template/betterjson/LanguageConsistencyFixer.java

@@ -0,0 +1,43 @@
+package com.nedap.archie.template.betterjson;
+
+import com.nedap.archie.aom.Archetype;
+import com.nedap.archie.aom.Template;
+import com.nedap.archie.aom.TemplateOverlay;
+import com.nedap.archie.aom.terminology.ArchetypeTerm;
+import com.nedap.archie.aom.terminology.ArchetypeTerminology;
+import com.nedap.archie.archetypevalidator.ErrorType;
+
+import java.util.List;
+
+public class LanguageConsistencyFixer {
+
+    public void fixLanguageConsistency(Archetype archetype) {
+
+        fixInner(archetype);
+        if(archetype instanceof Template) {
+            Template template = (Template) archetype;
+            for(TemplateOverlay overlay:template.getTemplateOverlays()) {
+                fixLanguageConsistency(overlay);
+            }
+        }
+
+    }
+
+    private void fixInner(Archetype archetype) {
+
+        List<String> codes = archetype.getTerminology().allCodes();
+        for(String code:codes) {
+            for (String language : archetype.getTerminology().getTermDefinitions().keySet()) {
+                if(!archetype.getTerminology().getTermDefinitions().get(language).containsKey(code)) {
+
+                    ArchetypeTerm term = new ArchetypeTerm();
+                    term.setCode(code);
+                    term.setText("translation missing");
+                    term.setDescription("translation missing");
+                    archetype.getTerminology().getTermDefinitions().get(language).put(code, term);
+                }
+            }
+        }
+    }
+
+}