BLZG-2083 str() produces NotMaterializedException when using group by/sample

bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java

@@ -34,6 +34,7 @@
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.log4j.Logger;
@@ -1996,7 +1997,25 @@ public static boolean requiresMaterialization(final IConstraint c) {
     public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
         final BOp c, final Set<IVariable<IV>> terms) {
 
-        return gatherVarsToMaterialize(c, terms, false /* includeVarsInAnnotations */);
+        return gatherVarsToMaterialize(c, terms, null /*varMap*/, false /* includeVarsInAnnotations */);
+    }
+
+    /**
+     * Static helper used to determine materialization requirements.
+     */
+    public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
+        final BOp c, final Set<IVariable<IV>> terms, final Map<IVariable<?>, IValueExpression<?>> varMap) {
+
+        return gatherVarsToMaterialize(c, terms, varMap, false /* includeVarsInAnnotations */);
+    }
+
+    /**
+     * Static helper used to determine materialization requirements.
+     */
+    public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
+        final BOp c, final Set<IVariable<IV>> terms, final boolean includeVarsInAnnotations) {
+
+        return gatherVarsToMaterialize(c, terms, null /*varMap*/, includeVarsInAnnotations);
     }
 
     /**
@@ -2011,7 +2030,7 @@ public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
      */
     @SuppressWarnings("rawtypes")
     public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
-            final BOp c, final Set<IVariable<IV>> terms, final boolean includeVarsInAnnotations) {
+            final BOp c, final Set<IVariable<IV>> terms, final Map<IVariable<?>, IValueExpression<?>> varMap, final boolean includeVarsInAnnotations) {
 
         boolean materialize = false;
         boolean always = false;
@@ -2027,9 +2046,31 @@ public static INeedsMaterialization.Requirement gatherVarsToMaterialize(
             if (bop instanceof INeedsMaterialization) {
 
                 final INeedsMaterialization bop2 = (INeedsMaterialization) bop;
-
+                               
                 final Set<IVariable<IV>> t = getVarsFromArguments(bop);
 
+                // https://jira.blazegraph.com/browse/BLZG-2083 (str() produces NotMaterializedException when using group by/sample)
+                // NotMaterializedException can be thrown by a function referenced in group by, to avoid that all variables referenced
+                // in group by expressions should be materialized, as MemoryGroupByOp needs real IVs. To avoid excessive adding of
+                // unneeded variables, varMap is provided as a reference of variables which might need materialization.
+                if (varMap != null) {
+
+                    for (IVariable<IV> key : t) {
+
+                        IValueExpression expr = varMap.get(key);
+
+                        if (expr != null) {
+
+                            Set<IVariable<IV>> vars = getVarsFromArguments(expr);
+
+                            t.addAll(vars);
+
+                        }
+
+                     }
+
+                 }
+
                 if (t.size() > 0) {
 
                     terms.addAll(t);

bigdata-core/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java

@@ -2,6 +2,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -4597,6 +4598,13 @@ private static final PipelineOp addAggregation(PipelineOp left,
         final IGroupByRewriteState groupByRewrite = new GroupByRewriter(
                 groupByState);
 
+        // https://jira.blazegraph.com/browse/BLZG-2083 (str() produces NotMaterializedException when using group by/sample)
+        // NotMaterializedException can be thrown by a function referenced in group by, in this case MemoryGroupByOp needs IV,
+        // but IVs is not materialized yet. So we need to collect real bindingset variables
+        // from referenced expressions and pass them out to addMaterializationSteps2.
+
+        final Map<IVariable<?>, IValueExpression<?>> varsToExprMap = getVarsToExprMap(projection);
+
         // The query hints are taken from the PROJECTION.
         final Properties queryHints = projection.getQueryHints();
 
@@ -4645,7 +4653,7 @@ private static final PipelineOp addAggregation(PipelineOp left,
 
                 } else {
 
-                    StaticAnalysis.gatherVarsToMaterialize(expr, vars);
+                    StaticAnalysis.gatherVarsToMaterialize(expr, vars, varsToExprMap);
 
                 }
 
@@ -4675,7 +4683,7 @@ private static final PipelineOp addAggregation(PipelineOp left,
 
                 } else {
 
-                    StaticAnalysis.gatherVarsToMaterialize(expr, vars);
+                    StaticAnalysis.gatherVarsToMaterialize(expr, vars, varsToExprMap);
 
                 }
 
@@ -4752,7 +4760,31 @@ private static final PipelineOp addAggregation(PipelineOp left,
 
     }
 
-    /**
+   /**
+    * Collect all variables defined in projection node 
+    */
+   private static Map<IVariable<?>, IValueExpression<?>> getVarsToExprMap(final ProjectionNode projection) {
+
+    final Map<IVariable<?>, IValueExpression<?>> map = new HashMap<>();
+
+       for ( IValueExpression<?> expr : projection.getValueExpressions()) {
+
+           if (expr instanceof Bind) {
+
+               final IVariable<?> var = ((Bind<?>) expr).getVar();
+               final IValueExpression<?> val = ((Bind<?>) expr).getExpr();
+
+               map.put(var, val);
+
+           }
+
+       }
+
+       return map;
+
+   }
+
+	/**
      * Add an ORDER BY operator.
      */
     @SuppressWarnings({ "unchecked", "rawtypes" })

bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java

@@ -124,6 +124,7 @@ public static Test suite() {
         //suite.addTestSuite(com.bigdata.rdf.sail.TestTicket1939.class);
         suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2043.class);
         suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2043b.class);
+        suite.addTestSuite(com.bigdata.rdf.sail.TestTicket2083.class);
 
         suite.addTestSuite(com.bigdata.rdf.sail.webapp.TestBLZG1943.class);
 

bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.java

@@ -0,0 +1,133 @@
+/**
+Copyright (C) SYSTAP, LLC DBA Blazegraph 2011.  All rights reserved.
+
+Contact:
+     SYSTAP, LLC DBA Blazegraph
+     2501 Calvert ST NW #106
+     Washington, DC 20008
+     licenses@blazegraph.com
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package com.bigdata.rdf.sail;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.repository.RepositoryException;
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.RDFParseException;
+
+/**
+ * Unit test template for use in submission of bugs.
+ * <p>
+ * This test case will delegate to an underlying backing store. You can specify
+ * this store via a JVM property as follows:
+ * <code>-DtestClass=com.bigdata.rdf.sail.TestBigdataSailWithQuads</code>
+ * <p>
+ * There are three possible configurations for the testClass:
+ * <ul>
+ * <li>com.bigdata.rdf.sail.TestBigdataSailWithQuads (quads mode)</li>
+ * <li>com.bigdata.rdf.sail.TestBigdataSailWithoutSids (triples mode)</li>
+ * <li>com.bigdata.rdf.sail.TestBigdataSailWithSids (SIDs mode)</li>
+ * </ul>
+ * <p>
+ * The default for triples and SIDs mode is for inference with truth maintenance
+ * to be on. If you would like to turn off inference, make sure to do so in
+ * {@link #getProperties()}.
+ * 
+ * @author <a href="mailto:mrpersonick@users.sourceforge.net">Mike Personick</a>
+ * @version $Id$
+ * 
+ * @see https://jira.blazegraph.com/browse/BLZG-2082
+ * 		
+ */
+public class TestTicket2083 extends QuadsTestCase {
+
+    public TestTicket2083() {
+	}
+
+	public TestTicket2083(String arg0) {
+		super(arg0);
+	}
+
+	@Override
+	public Properties getProperties() {
+		Properties properties = super.getProperties();
+		return properties;
+	}
+
+	public void testBug() throws Exception {
+
+		final BigdataSail sail = getSail();
+		try {
+			executeQuery(new BigdataSailRepository(sail));
+		} finally {
+			sail.__tearDownUnitTest();
+		}
+	}
+
+	private void executeQuery(final BigdataSailRepository repo)
+			throws RepositoryException, MalformedQueryException,
+			QueryEvaluationException, RDFParseException, IOException {
+		try {
+			repo.initialize();
+			final BigdataSailRepositoryConnection conn = repo.getConnection();
+			conn.setAutoCommit(false);
+			try {
+				conn.add(getClass().getResourceAsStream("TestTicket2083.n3"), "",
+						RDFFormat.TURTLE);
+				conn.commit();
+
+				final String query = "PREFIX wd: <http://wd/>\n" + 
+						"PREFIX wdt: <http://wdt/>\n" +
+						"SELECT ?id\r\n" + 
+						"(SAMPLE(?img) as ?pic)\r\n" + 
+						"(str(?pic) as ?description)\r\n" + 
+						"WHERE {\r\n" + 
+						"  ?id wdt:P31 wd:Q35657 .\r\n" + 
+						"  ?id wdt:P6 ?head .\r\n" + 
+						"  OPTIONAL {\r\n" + 
+						"    ?head wdt:P18 ?img\r\n" + 
+						"  }\r\n" + 
+						"} GROUP BY ?id ?head ?description ";
+				final TupleQuery q = conn.prepareTupleQuery(QueryLanguage.SPARQL,
+						query);
+				final TupleQueryResult tqr = q.evaluate();
+				int cnt = 0;
+				while (tqr.hasNext()) {
+				    BindingSet bindings = tqr.next();
+				    cnt++;
+//				    if(log.isInfoEnabled())
+				        System.out.println("bindings="+bindings);
+				}
+				tqr.close();
+				assertEquals(1, cnt);
+
+			} finally {
+				conn.close();
+			}
+		} finally {
+			repo.shutDown();
+		}
+	}
+
+}

bigdata-sails-test/src/test/java/com/bigdata/rdf/sail/TestTicket2083.n3

@@ -0,0 +1,6 @@
+@prefix wd: <http://wd/> .
+@prefix wdt: <http://wdt/> .
+
+wd:1 wdt:P31 wd:Q35657 .
+wd:1 wdt:P6 wd:head .
+wd:head wdt:P18 <http://site/img.png> .