Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,28 @@ object CHConfig extends ConfigRegistry {
.doc("Enable local cache for CH backend.")
.booleanConf
.createWithDefault(false)

val EXTENDED_COLUMNAR_TRANSFORM_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
.internal()
.withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
.doc("A comma-separated list of classes for the extended columnar transform rules.")
.stringConf
.createWithDefaultString("")

val EXTENDED_COLUMNAR_POST_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
.internal()
.doc("A comma-separated list of classes for the extended columnar post rules.")
.stringConf
.createWithDefaultString("")

val EXTENDED_EXPRESSION_TRAN_CONF =
buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
.internal()
.doc("A class for the extended expressions transformer.")
.stringConf
.createWithDefaultString("")
}

class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
Expand All @@ -128,6 +150,14 @@ class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
getConf(ENABLE_CH_REWRITE_DATE_CONVERSION)

def enableGlutenLocalFileCache: Boolean = getConf(ENABLE_GLUTEN_LOCAL_FILE_CACHE)

// A comma-separated list of classes for the extended columnar pre rules
def extendedColumnarTransformRules: String = getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)

// A comma-separated list of classes for the extended columnar post rules
def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)

def extendedExpressionTransformer: String = getConf(EXTENDED_EXPRESSION_TRAN_CONF)
}

object GlutenObjectStorageConfig {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class CHListenerApi extends ListenerApi with Logging {
initialize(pc.conf, isDriver = true)

val expressionExtensionTransformer = ExpressionUtil.extendedExpressionTransformer(
pc.conf.get(GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF)
pc.conf.get(CHConfig.EXTENDED_EXPRESSION_TRAN_CONF)
)
if (expressionExtensionTransformer != null) {
ExpressionExtensionTrait.registerExpressionExtension(expressionExtensionTransformer)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ object CHRuleApi {
c =>
intercept(
SparkPlanRules.extendedColumnarRule(
new GlutenConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
new CHConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
injector.injectPostTransform(_ => CollectLimitTransformerRule())
injector.injectPostTransform(_ => CollectTailTransformerRule())
injector.injectPostTransform(c => InsertTransitions.create(c.outputsColumnar, CHBatchType))
Expand All @@ -153,8 +153,8 @@ object CHRuleApi {
injector.injectPost(
c =>
intercept(
SparkPlanRules.extendedColumnarRule(
new GlutenConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
SparkPlanRules
.extendedColumnarRule(new CHConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
injector.injectPost(c => GlutenNoopWriterRule.apply(c.session))

// Gluten columnar: Final rules.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.gluten.execution.extension

import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.backendsapi.clickhouse.CHConfig
import org.apache.gluten.execution._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.SubstraitPlanPrinterUtil
Expand All @@ -33,7 +33,7 @@ class GlutenCustomAggExpressionSuite extends ParquetSuite {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
"org.apache.gluten.execution.extension.CustomAggExpressionTransformer")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.extension

import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.backendsapi.clickhouse.CHConfig
import org.apache.gluten.execution.{GlutenClickHouseWholeStageTransformerSuite, ProjectExecTransformer}
import org.apache.gluten.expression.ExpressionConverter

Expand Down Expand Up @@ -88,7 +88,7 @@ class GlutenClickhouseCustomerExpressionTransformerSuite
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
"org.apache.spark.sql.extension.CustomerExpressionTransformer")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,14 +286,6 @@ class GlutenConfig(conf: SQLConf) extends GlutenCoreConfig(conf) {

def softAffinityLogLevel: String = getConf(SOFT_AFFINITY_LOG_LEVEL)

// A comma-separated list of classes for the extended columnar pre rules
def extendedColumnarTransformRules: String = getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)

// A comma-separated list of classes for the extended columnar post rules
def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)

def extendedExpressionTransformer: String = getConf(EXTENDED_EXPRESSION_TRAN_CONF)

def smallFileThreshold: Double = getConf(SMALL_FILE_THRESHOLD)

def expressionBlacklist: Set[String] = {
Expand Down Expand Up @@ -1314,31 +1306,6 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)

// FIXME: This only works with CH backend.
val EXTENDED_COLUMNAR_TRANSFORM_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
.internal()
.withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
.doc("A comma-separated list of classes for the extended columnar transform rules.")
.stringConf
.createWithDefaultString("")

// FIXME: This only works with CH backend.
val EXTENDED_COLUMNAR_POST_RULES =
buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
.internal()
.doc("A comma-separated list of classes for the extended columnar post rules.")
.stringConf
.createWithDefaultString("")

// FIXME: This only works with CH backend.
val EXTENDED_EXPRESSION_TRAN_CONF =
buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
.internal()
.doc("A class for the extended expressions transformer.")
.stringConf
.createWithDefaultString("")

val EXPRESSION_BLACK_LIST =
buildConf("spark.gluten.expression.blacklist")
.doc("A black list of expression to skip transform, multiple values separated by commas.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait

class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
// These configs only take effect on ClickHouse backend.
private val ExtendedColumnarTransformRulesKey =
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
private val ExtendedColumnarPostRulesKey =
"spark.gluten.sql.columnar.extended.columnar.post.rules"

override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
.set(ExtendedColumnarPostRulesKey, "")
}

testGluten("test customer column rules") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait

class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
// These configs only take effect on ClickHouse backend.
private val ExtendedColumnarTransformRulesKey =
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
private val ExtendedColumnarPostRulesKey =
"spark.gluten.sql.columnar.extended.columnar.post.rules"

override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
.set(ExtendedColumnarPostRulesKey, "")
}

testGluten("test customer column rules") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait

class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
// These configs only take effect on ClickHouse backend.
private val ExtendedColumnarTransformRulesKey =
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
private val ExtendedColumnarPostRulesKey =
"spark.gluten.sql.columnar.extended.columnar.post.rules"

override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
.set(ExtendedColumnarPostRulesKey, "")
}

testGluten("test customer column rules") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait

class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
// These configs only take effect on ClickHouse backend.
private val ExtendedColumnarTransformRulesKey =
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
private val ExtendedColumnarPostRulesKey =
"spark.gluten.sql.columnar.extended.columnar.post.rules"

override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
.set(ExtendedColumnarPostRulesKey, "")
}

testGluten("test customer column rules") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait

class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
// These configs only take effect on ClickHouse backend.
private val ExtendedColumnarTransformRulesKey =
"spark.gluten.sql.columnar.extended.columnar.transform.rules"
private val ExtendedColumnarPostRulesKey =
"spark.gluten.sql.columnar.extended.columnar.post.rules"

override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
.set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
.set(ExtendedColumnarPostRulesKey, "")
}

testGluten("test customer column rules") {
Expand Down
Loading