diff --git a/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java b/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java index 7d1bf13930..2a4db8a950 100644 --- a/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java +++ b/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java @@ -34,6 +34,7 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.log4j.Logger; @@ -1996,7 +1997,25 @@ public static boolean requiresMaterialization(final IConstraint c) { public static INeedsMaterialization.Requirement gatherVarsToMaterialize( final BOp c, final Set> terms) { - return gatherVarsToMaterialize(c, terms, false /* includeVarsInAnnotations */); + return gatherVarsToMaterialize(c, terms, null /*varMap*/, false /* includeVarsInAnnotations */); + } + + /** + * Static helper used to determine materialization requirements. + */ + public static INeedsMaterialization.Requirement gatherVarsToMaterialize( + final BOp c, final Set> terms, final Map, IValueExpression> varMap) { + + return gatherVarsToMaterialize(c, terms, varMap, false /* includeVarsInAnnotations */); + } + + /** + * Static helper used to determine materialization requirements. + */ + public static INeedsMaterialization.Requirement gatherVarsToMaterialize( + final BOp c, final Set> terms, final boolean includeVarsInAnnotations) { + + return gatherVarsToMaterialize(c, terms, null /*varMap*/, includeVarsInAnnotations); } /** @@ -2011,7 +2030,7 @@ public static INeedsMaterialization.Requirement gatherVarsToMaterialize( */ @SuppressWarnings("rawtypes") public static INeedsMaterialization.Requirement gatherVarsToMaterialize( - final BOp c, final Set> terms, final boolean includeVarsInAnnotations) { + final BOp c, final Set> terms, final Map, IValueExpression> varMap, final boolean includeVarsInAnnotations) { boolean materialize = false; boolean always = false; @@ -2027,9 +2046,31 @@ public static INeedsMaterialization.Requirement gatherVarsToMaterialize( if (bop instanceof INeedsMaterialization) { final INeedsMaterialization bop2 = (INeedsMaterialization) bop; - + final Set> t = getVarsFromArguments(bop); + // https://jira.blazegraph.com/browse/BLZG-2083 (str() produces NotMaterializedException when using group by/sample) + // NotMaterializedException can be thrown by a function referenced in group by, to avoid that all variables referenced + // in group by expressions should be materialized, as MemoryGroupByOp needs real IVs. To avoid excessive adding of + // unneeded variables, varMap is provided as a reference of variables which might need materialization. + if (varMap != null) { + + for (IVariable key : t) { + + IValueExpression expr = varMap.get(key); + + if (expr != null) { + + Set> vars = getVarsFromArguments(expr); + + t.addAll(vars); + + } + + } + + } + if (t.size() > 0) { terms.addAll(t); diff --git a/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java b/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java index 97ee814861..79f5cf9ac4 100644 --- a/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java +++ b/bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -4597,6 +4598,13 @@ private static final PipelineOp addAggregation(PipelineOp left, final IGroupByRewriteState groupByRewrite = new GroupByRewriter( groupByState); + // https://jira.blazegraph.com/browse/BLZG-2083 (str() produces NotMaterializedException when using group by/sample) + // NotMaterializedException can be thrown by a function referenced in group by, in this case MemoryGroupByOp needs IV, + // but IVs is not materialized yet. So we need to collect real bindingset variables + // from referenced expressions and pass them out to addMaterializationSteps2. + + final Map, IValueExpression> varsToExprMap = getVarsToExprMap(projection); + // The query hints are taken from the PROJECTION. final Properties queryHints = projection.getQueryHints(); @@ -4645,7 +4653,7 @@ private static final PipelineOp addAggregation(PipelineOp left, } else { - StaticAnalysis.gatherVarsToMaterialize(expr, vars); + StaticAnalysis.gatherVarsToMaterialize(expr, vars, varsToExprMap); } @@ -4675,7 +4683,7 @@ private static final PipelineOp addAggregation(PipelineOp left, } else { - StaticAnalysis.gatherVarsToMaterialize(expr, vars); + StaticAnalysis.gatherVarsToMaterialize(expr, vars, varsToExprMap); } @@ -4752,7 +4760,31 @@ private static final PipelineOp addAggregation(PipelineOp left, } - /** + /** + * Collect all variables defined in projection node + */ + private static Map, IValueExpression> getVarsToExprMap(final ProjectionNode projection) { + + final Map, IValueExpression> map = new HashMap<>(); + + for ( IValueExpression expr : projection.getValueExpressions()) { + + if (expr instanceof Bind) { + + final IVariable var = ((Bind) expr).getVar(); + final IValueExpression val = ((Bind) expr).getExpr(); + + map.put(var, val); + + } + + } + + return map; + + } + + /** * Add an ORDER BY operator. */ @SuppressWarnings({ "unchecked", "rawtypes" }) diff --git a/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java index 5d5edf2291..f0627bcc0c 100644 --- a/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java +++ b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java @@ -124,6 +124,7 @@ public static Test suite() { //suite.addTestSuite(com.bigdata.rdf.sail.TestTicket1939.class); suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2043.class); suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2043b.class); + suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2083.class); suite.addTestSuite(com.bigdata.rdf.sail.webapp.TestBLZG1943.class); diff --git a/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.java b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.java new file mode 100644 index 0000000000..177c032e96 --- /dev/null +++ b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.java @@ -0,0 +1,133 @@ +/** +Copyright (C) SYSTAP, LLC DBA Blazegraph 2011. All rights reserved. + +Contact: + SYSTAP, LLC DBA Blazegraph + 2501 Calvert ST NW #106 + Washington, DC 20008 + licenses@blazegraph.com + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package com.bigdata.rdf.sail; + +import java.io.IOException; +import java.util.Properties; + +import org.openrdf.query.BindingSet; +import org.openrdf.query.MalformedQueryException; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.QueryLanguage; +import org.openrdf.query.TupleQuery; +import org.openrdf.query.TupleQueryResult; +import org.openrdf.repository.RepositoryException; +import org.openrdf.rio.RDFFormat; +import org.openrdf.rio.RDFParseException; + +/** + * Unit test template for use in submission of bugs. + *

+ * This test case will delegate to an underlying backing store. You can specify + * this store via a JVM property as follows: + * -DtestClass=com.bigdata.rdf.sail.TestBigdataSailWithQuads + *

+ * There are three possible configurations for the testClass: + *

    + *
  • com.bigdata.rdf.sail.TestBigdataSailWithQuads (quads mode)
    • + *
  • com.bigdata.rdf.sail.TestBigdataSailWithoutSids (triples mode)
    • + *
  • com.bigdata.rdf.sail.TestBigdataSailWithSids (SIDs mode)
    • + *
+ *

+ * The default for triples and SIDs mode is for inference with truth maintenance + * to be on. If you would like to turn off inference, make sure to do so in + * {@link #getProperties()}. + * + * @author Mike Personick + * @version $Id$ + * + * @see https://jira.blazegraph.com/browse/BLZG-2082 + * + */ +public class TestTicket2083 extends QuadsTestCase { + + public TestTicket2083() { + } + + public TestTicket2083(String arg0) { + super(arg0); + } + + @Override + public Properties getProperties() { + Properties properties = super.getProperties(); + return properties; + } + + public void testBug() throws Exception { + + final BigdataSail sail = getSail(); + try { + executeQuery(new BigdataSailRepository(sail)); + } finally { + sail.__tearDownUnitTest(); + } + } + + private void executeQuery(final BigdataSailRepository repo) + throws RepositoryException, MalformedQueryException, + QueryEvaluationException, RDFParseException, IOException { + try { + repo.initialize(); + final BigdataSailRepositoryConnection conn = repo.getConnection(); + conn.setAutoCommit(false); + try { + conn.add(getClass().getResourceAsStream("TestTicket2083.n3"), "", + RDFFormat.TURTLE); + conn.commit(); + + final String query = "PREFIX wd: \n" + + "PREFIX wdt: \n" + + "SELECT ?id\r\n" + + "(SAMPLE(?img) as ?pic)\r\n" + + "(str(?pic) as ?description)\r\n" + + "WHERE {\r\n" + + " ?id wdt:P31 wd:Q35657 .\r\n" + + " ?id wdt:P6 ?head .\r\n" + + " OPTIONAL {\r\n" + + " ?head wdt:P18 ?img\r\n" + + " }\r\n" + + "} GROUP BY ?id ?head ?description "; + final TupleQuery q = conn.prepareTupleQuery(QueryLanguage.SPARQL, + query); + final TupleQueryResult tqr = q.evaluate(); + int cnt = 0; + while (tqr.hasNext()) { + BindingSet bindings = tqr.next(); + cnt++; +// if(log.isInfoEnabled()) + System.out.println("bindings="+bindings); + } + tqr.close(); + assertEquals(1, cnt); + + } finally { + conn.close(); + } + } finally { + repo.shutDown(); + } + } + +} diff --git a/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.n3 b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.n3 new file mode 100644 index 0000000000..2adc6bf063 --- /dev/null +++ b/bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.n3 @@ -0,0 +1,6 @@ +@prefix wd: . +@prefix wdt: . + +wd:1 wdt:P31 wd:Q35657 . +wd:1 wdt:P6 wd:head . +wd:head wdt:P18 .