diff --git a/.travis.yml b/.travis.yml index a0217279..19e25c84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ branches: script: # Print output every minute to avoid travis timeout - while sleep 1m; do echo "=====[ $SECONDS seconds elapsed -- still running ]====="; done & - - ./gradlew build -s && ./gradlew -p transportable-udfs-examples build -s && ./gradlew ciPerformRelease -s + # With the exception of release commands, all build logic goes in travis-build.sh + - ./travis-build.sh && ./gradlew ciPerformRelease -s # Killing background sleep loop - kill %1 diff --git a/build.gradle b/build.gradle index c737741e..eb5d6ea7 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,8 @@ buildscript { } plugins { - id "org.shipkit.java" version "2.0.31" + id "org.shipkit.java" version "2.3.4" + id "checkstyle" } allprojects { @@ -48,11 +49,24 @@ subprojects { strictCheck true } + configurations { + all { + // Transport as a library should only expose slf4j-api to its consumers and should not keep any SLF4J bindings + // in its dependency graph + // Quote from http://www.slf4j.org/faq.html#maven2 + // "Thus, as far as your users are concerned you are exporting slf4j-api as a transitive dependency of your + // library, but not any SLF4J-binding or any underlying logging system." + // Our dependencies (e.g. hadoop-common) do bring in slf4j-log4j12 in their dependency graph so we exclude them + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + } + } + plugins.withType(JavaPlugin) { project.apply(plugin: 'checkstyle') dependencies { testCompile 'org.testng:testng:6.11' + testCompile 'org.slf4j:slf4j-simple:1.7.25' } test { @@ -60,7 +74,9 @@ subprojects { } checkstyle { - configFile = file("${rootDir}/gradle/checkstyle/linkedin-checkstyle.xml") + configFile = file("${rootDir}/gradle/checkstyle/checkstyle.xml") + configProperties = ['config_loc' : "${rootDir}/gradle/checkstyle/"] + toolVersion '8.23' } } diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index ced4d3d5..c6b83602 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -10,8 +10,8 @@ subprojects { url "https://conjars.org/repo" } } - project.ext.setProperty('presto-version', '319') - project.ext.setProperty('airlift-slice-version', '0.33') + project.ext.setProperty('presto-version', '333') + project.ext.setProperty('airlift-slice-version', '0.38') project.ext.setProperty('spark-group', 'org.apache.spark') project.ext.setProperty('spark-version', '2.3.0') } diff --git a/docs/release-notes.md b/docs/release-notes.md index cb7b2b3c..11d75db1 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,76 @@ *Release notes were automatically generated by [Shipkit](http://shipkit.org/)* +#### 0.0.61 + - 2020-11-26 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.60...v0.0.61) by [Carl Steinbach](https://github.com/cwsteinbach) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.61-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.61) + - Fail build if there are checkstyle violations [(#64)](https://github.com/linkedin/transport/pull/64) + +#### 0.0.60 + - 2020-11-18 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.59...v0.0.60) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.60-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.60) + - Add Avro ENUM read support. [(#62)](https://github.com/linkedin/transport/pull/62) + +#### 0.0.59 + - 2020-11-10 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.58...v0.0.59) by [curtiscwang](https://github.com/curtiscwang) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.59-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.59) + - Support conversion of String type to Utf8 in AvroWrapper [(#61)](https://github.com/linkedin/transport/pull/61) + +#### 0.0.58 + - 2020-11-04 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.57...v0.0.58) by [Khai Tran](https://github.com/khaitranq) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.58-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.58) + - Support simple Avro Union schema [(#60)](https://github.com/linkedin/transport/pull/60) + +#### 0.0.57 + - 2020-09-23 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.56...v0.0.57) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.57-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.57) + - Create a MutableMap when accessing keySet and values in SparkMap [(#58)](https://github.com/linkedin/transport/pull/58) + +#### 0.0.56 + - 2020-08-17 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.55...v0.0.56) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.56-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.56) + - Fix test SQL generation for binary inputs [(#55)](https://github.com/linkedin/transport/pull/55) + +#### 0.0.55 + - 2020-08-13 - [3 commits](https://github.com/linkedin/transport/compare/v0.0.54...v0.0.55) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.55-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.55) + - Bump shipkit version [(#54)](https://github.com/linkedin/transport/pull/54) + - Remove slf4j-log4j12 from Transport dependency graph [(#51)](https://github.com/linkedin/transport/pull/51) + - Hive: Struct data should not be converted to object array during StdStruct creation [(#50)](https://github.com/linkedin/transport/pull/50) + +#### 0.0.54 + - 2020-08-10 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.53...v0.0.54) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.54-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.54) + - Publish Presto thin jar which allows consumers to control dependency graph [(#49)](https://github.com/linkedin/transport/pull/49) + +#### 0.0.53 + - 2020-06-30 - [2 commits](https://github.com/linkedin/transport/compare/v0.0.52...v0.0.53) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.53-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.53) + - No pull requests referenced in commit messages. + +#### 0.0.52 + - 2020-06-26 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.51...v0.0.52) by [John Joyce](https://github.com/jjoyce0510) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.52-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.52) + - No pull requests referenced in commit messages. + +#### 0.0.51 + - 2020-06-23 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.50...v0.0.51) by [Khai Tran](https://github.com/khaitranq) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.51-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.51) + - Add support for StdFloat, StdDouble, and StdBinary [(#46)](https://github.com/linkedin/transport/pull/46) + +#### 0.0.50 + - 2020-05-08 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.49...v0.0.50) by [Xingyuan Lin](https://github.com/lxynov) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.50-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.50) + - Upgrade to PrestoSQL 333 [(#45)](https://github.com/linkedin/transport/pull/45) + +#### 0.0.49 + - 2020-04-30 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.48...v0.0.49) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.49-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.49) + - Presto: Make ScalarFunctionImplementation state independent of StdUdfWrapper [(#44)](https://github.com/linkedin/transport/pull/44) + +#### 0.0.48 + - 2020-04-14 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.47...v0.0.48) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.48-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.48) + - Presto: Pass custom configuration object when using FileSystemUtils [(#43)](https://github.com/linkedin/transport/pull/43) + +#### 0.0.47 + - 2020-04-01 - [2 commits](https://github.com/linkedin/transport/compare/v0.0.46...v0.0.47) by [Suren Nihalani](https://github.com/SurenNihalani) (1), [Sushant Raikar](https://github.com/HotSushi) (1) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.47-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.47) + - FileSystemUtils: remove an unreliable check for unit testing [(#42)](https://github.com/linkedin/transport/pull/42) + - Fixed Presto UDF patch's broken link [(#41)](https://github.com/linkedin/transport/pull/41) + +#### 0.0.46 + - 2020-02-03 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.45...v0.0.46) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.46-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.46) + - Disable Jacoco for platform tests [(#37)](https://github.com/linkedin/transport/pull/37) + +#### 0.0.45 + - 2020-01-14 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.44...v0.0.45) by [Suren Nihalani](https://github.com/SurenNihalani) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.45-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.45) + - Allow for absolute paths instead of assuming defaultFS in UDF's required files [(#33)](https://github.com/linkedin/transport/pull/33) + #### 0.0.44 - 2019-12-13 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.43...v0.0.44) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.44-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.44) - Remove log4j properties from main sources [(#32)](https://github.com/linkedin/transport/pull/32) diff --git a/docs/transport-udfs-api.md b/docs/transport-udfs-api.md index fadaa75f..deecd1cb 100644 --- a/docs/transport-udfs-api.md +++ b/docs/transport-udfs-api.md @@ -9,7 +9,8 @@ The `StdType` interface is the parent class of all type objects that are used to describe the schema of the data objects that can be manipulated by `StdUDFs`. Sub-interfaces of this interface include `StdIntegerType`, `StdBooleanType`, `StdLongType`, `StdStringType`, -`StdArrayType`, `StdMapType`, `StdStructType`. Each sub-interface is +`StdDoubleType`, `StdFloatType`, `StdBinaryType`, `StdArrayType`, +`StdMapType`, and `StdStructType`. Each sub-interface is defined by methods that are specific to the corresponding type. For example, `StdMapType` interface is defined by the two methods shown below. The `keyType()` and `valueType()` methods can be used to obtain @@ -39,9 +40,10 @@ public interface StdStructType extends StdType { manipulated by Transport UDFs. As a top-level interface, `StdData` itself does not contain any methods. A number of type-specific interfaces extend `StdData`, such as `StdInteger`, `StdLong`, -`StdBoolean`, `StdString`, `StdArray`, `StdMap`, `StdStruct` to -represent `INTEGER`, `LONG`, `BOOLEAN`, `VARCHAR`, `ARRAY`, `MAP`, -`STRUCT` SQL types respectively. Each of those interfaces exposes +`StdBoolean`, `StdString`, `StdDouble`, `StdFloat`, `StdBinary`, +`StdArray`, `StdMap`, and `StdStruct` to represent `INTEGER`, +`LONG`, `BOOLEAN`, `VARCHAR`, `DOUBLE`, `REAL`, `VARBINARY`, `ARRAY`, `MAP`, +and `STRUCT` SQL types respectively. Each of those interfaces exposes operations that can manipulate that type of data. For example, `StdMap` interface is defined by the following methods: @@ -108,6 +110,12 @@ definition: is StdInteger. * `"boolean"`: to represent SQL Boolean type. The respective Standard Type is StdBoolean. +* `"double"`: to represent SQL Double type. The respective Standard + Type is StdDouble. +* `"real"`: to represent SQL Real type. The respective Standard + Type is StdFloat. +* `"varbinary"`: to represent SQL Binary type. The respective Standard + Type is StdBinary. * `"array(T)"`: to represent SQL Array type, with elements of type T. The respective Standard Type is StdArray. * `"map(K,V)"`: to represent SQL Map type, with keys of type K and @@ -132,6 +140,9 @@ public interface StdFactory { StdLong createLong(long value); StdBoolean createBoolean(boolean value); StdString createString(String value); + StdDouble createDouble(double value); + StdFloat createFloat(float value); + StdBinary createBinary(ByteBuffer value); StdArray createArray(StdType stdType, int expectedSize); StdArray createArray(StdType stdType); StdMap createMap(StdType stdType); diff --git a/transportable-udfs-documentation/transport-udfs-presto.patch b/docs/transport-udfs-presto.patch similarity index 58% rename from transportable-udfs-documentation/transport-udfs-presto.patch rename to docs/transport-udfs-presto.patch index 3fa1a9b8..c29b1dd6 100644 --- a/transportable-udfs-documentation/transport-udfs-presto.patch +++ b/docs/transport-udfs-presto.patch @@ -1,28 +1,5 @@ -diff --git a/presto-main/pom.xml b/presto-main/pom.xml -index 83a20a23fe..02afa5310f 100644 ---- a/presto-main/pom.xml -+++ b/presto-main/pom.xml -@@ -424,6 +424,18 @@ - test - - -+ -+ org.apache.maven.plugins -+ maven-jar-plugin -+ 2.2 -+ -+ -+ -+ test-jar -+ -+ -+ -+ - - - diff --git a/presto-main/src/main/java/io/prestosql/server/PluginManager.java b/presto-main/src/main/java/io/prestosql/server/PluginManager.java -index f02ceeab03..88de943bed 100644 +index abcd001031..053c17aeed 100644 --- a/presto-main/src/main/java/io/prestosql/server/PluginManager.java +++ b/presto-main/src/main/java/io/prestosql/server/PluginManager.java @@ -23,6 +23,7 @@ import io.prestosql.connector.ConnectorManager; @@ -31,17 +8,17 @@ index f02ceeab03..88de943bed 100644 import io.prestosql.metadata.MetadataManager; +import io.prestosql.metadata.SqlScalarFunction; import io.prestosql.security.AccessControlManager; + import io.prestosql.security.GroupProviderManager; import io.prestosql.server.security.PasswordAuthenticatorManager; - import io.prestosql.spi.Plugin; -@@ -52,6 +53,7 @@ import java.util.List; - import java.util.ServiceLoader; +@@ -54,6 +55,7 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; + import java.util.function.Supplier; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkState; import static io.prestosql.metadata.FunctionExtractor.extractFunctions; -@@ -62,8 +64,22 @@ import static java.util.Objects.requireNonNull; +@@ -64,8 +66,22 @@ import static java.util.Objects.requireNonNull; @ThreadSafe public class PluginManager { @@ -54,8 +31,8 @@ index f02ceeab03..88de943bed 100644 private static final ImmutableList SPI_PACKAGES = ImmutableList.builder() + // io.prestosql.metadata is required for SqlScalarFunction and FunctionRegistry classes + .add("io.prestosql.metadata.") -+ // io.prestosql.operator.scalar is required for ScalarFunctionImplementation -+ .add("io.prestosql.operator.scalar.") ++ // io.prestosql.operator. is required for ScalarFunctionImplementation and TypeSignatureParser ++ .add("io.prestosql.operator.") .add("io.prestosql.spi.") + // io.prestosql.type is required for TypeManager, and all supported types + .add("io.prestosql.type.") @@ -63,8 +40,8 @@ index f02ceeab03..88de943bed 100644 + .add("io.prestosql.util.") .add("com.fasterxml.jackson.annotation.") .add("io.airlift.slice.") - .add("io.airlift.units.") -@@ -155,7 +171,21 @@ public class PluginManager + .add("org.openjdk.jol.") +@@ -159,11 +175,22 @@ public class PluginManager { ServiceLoader serviceLoader = ServiceLoader.load(Plugin.class, pluginClassLoader); List plugins = ImmutableList.copyOf(serviceLoader); @@ -74,32 +51,21 @@ index f02ceeab03..88de943bed 100644 + pluginClassLoader); + List sqlScalarFunctions = ImmutableList.copyOf(sqlScalarFunctionsServiceLoader); + -+ checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", -+ Plugin.class.getName(), SqlScalarFunction.class.getName()); ++ checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", Plugin.class.getName(), SqlScalarFunction.class.getName()); + -+ installPlugins(plugins); -+ -+ registerSqlScalarFunctions(sqlScalarFunctions); -+ } -+ -+ private void installPlugins(List plugins) -+ { for (Plugin plugin : plugins) { log.info("Installing %s", plugin.getClass().getName()); - installPlugin(plugin); -@@ -215,6 +245,15 @@ public class PluginManager + installPlugin(plugin, pluginClassLoader::duplicate); } - } - -+ public void registerSqlScalarFunctions(List sqlScalarFunctions) -+ { ++ + for (SqlScalarFunction sqlScalarFunction : sqlScalarFunctions) { -+ log.info("Registering function %s(%s)", sqlScalarFunction.getSignature().getName(), sqlScalarFunction.getSignature().getArgumentTypes().stream().map(e -> e.toString()).collect( -+ Collectors.joining(", "))); ++ log.info("Registering function %s(%s)", ++ sqlScalarFunction.getFunctionMetadata().getSignature().getName(), ++ sqlScalarFunction.getFunctionMetadata().getSignature().getArgumentTypes().stream() ++ .map(e -> e.toString()) ++ .collect(Collectors.joining(", "))); + metadataManager.addFunctions(ImmutableList.of(sqlScalarFunction)); + } -+ } -+ - private URLClassLoader buildClassLoader(String plugin) - throws Exception - { + } + + public void installPlugin(Plugin plugin, Supplier duplicatePluginClassLoaderFactory) diff --git a/docs/using-transport-udfs.md b/docs/using-transport-udfs.md index 70f06edc..9b4e5f97 100644 --- a/docs/using-transport-udfs.md +++ b/docs/using-transport-udfs.md @@ -86,7 +86,7 @@ If the UDF class is `com.linkedin.transport.example.ExampleUDF` then the platfor Unlike Hive and Spark, Presto currently does not allow dynamically loading jar files once the Presto server has started. In Presto, the jar is deployed to the `plugin` directory. However, a small patch is required for the Presto engine to recognize the jar as a plugin, since the generated Presto UDFs implement the `SqlScalarFunction` API, which is currently not part of Presto's SPI architecture. -You can find the patch [here](transportable-udfs-documentation/transport-udfs-presto.patch) and apply it before deploying your UDFs jar to the Presto engine. +You can find the patch [here](transport-udfs-presto.patch) and apply it before deploying your UDFs jar to the Presto engine. 2. Call the UDF in a query To call the UDF, you will need to use the function name defined in the Transport UDF definition. diff --git a/gradle/checkstyle/linkedin-checkstyle.xml b/gradle/checkstyle/checkstyle.xml similarity index 98% rename from gradle/checkstyle/linkedin-checkstyle.xml rename to gradle/checkstyle/checkstyle.xml index 6af59c2e..a205c87e 100644 --- a/gradle/checkstyle/linkedin-checkstyle.xml +++ b/gradle/checkstyle/checkstyle.xml @@ -8,7 +8,7 @@ LinkedIn Java style. --> - + @@ -189,11 +189,11 @@ LinkedIn Java style. - --> + diff --git a/gradle/checkstyle/suppressions.xml b/gradle/checkstyle/suppressions.xml new file mode 100644 index 00000000..d4d3fa1a --- /dev/null +++ b/gradle/checkstyle/suppressions.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/TransportProcessor.java b/transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/TransportProcessor.java index 12e25002..30a110a8 100644 --- a/transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/TransportProcessor.java +++ b/transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/TransportProcessor.java @@ -130,10 +130,19 @@ private void processUDFClass(TypeElement udfClassElement) { udfClassElement ); } else { - String topLevelStdUdfClassName = - elementsOverridingTopLevelStdUDFMethods.iterator().next().getQualifiedName().toString(); + TypeElement topLevelStdUdfTypeElement = elementsOverridingTopLevelStdUDFMethods.iterator().next(); + String topLevelStdUdfClassName = topLevelStdUdfTypeElement.getQualifiedName().toString(); debug(String.format("TopLevelStdUDF class found: %s", topLevelStdUdfClassName)); + String udfClassName = udfClassElement.getQualifiedName().toString(); _transportUdfMetadata.addUDF(topLevelStdUdfClassName, udfClassElement.getQualifiedName().toString()); + _transportUdfMetadata.setClassNumberOfTypeParameters( + topLevelStdUdfClassName, + topLevelStdUdfTypeElement.getTypeParameters().size() + ); + _transportUdfMetadata.setClassNumberOfTypeParameters( + udfClassName, + udfClassElement.getTypeParameters().size() + ); } } diff --git a/transportable-udfs-annotation-processor/src/test/java/com/linkedin/transport/processor/TransportProcessorTest.java b/transportable-udfs-annotation-processor/src/test/java/com/linkedin/transport/processor/TransportProcessorTest.java index 0322f306..dd3c25c5 100644 --- a/transportable-udfs-annotation-processor/src/test/java/com/linkedin/transport/processor/TransportProcessorTest.java +++ b/transportable-udfs-annotation-processor/src/test/java/com/linkedin/transport/processor/TransportProcessorTest.java @@ -81,7 +81,7 @@ public void shouldNotContainMultipleOverridingsOfTopLevelStdUDFMethods1() throws .withErrorCount(1) .withErrorContaining(Constants.MORE_THAN_ONE_TYPE_OVERRIDING_ERROR) .in(forResource("udfs/UDFWithMultipleInterfaces1.java")) - .onLine(14) + .onLine(13) .atColumn(8); } @@ -96,7 +96,7 @@ public void shouldNotContainMultipleOverridingsOfTopLevelStdUDFMethods2() throws .withErrorCount(1) .withErrorContaining(Constants.MORE_THAN_ONE_TYPE_OVERRIDING_ERROR) .in(forResource("udfs/UDFWithMultipleInterfaces2.java")) - .onLine(13) + .onLine(12) .atColumn(8); } @@ -110,7 +110,7 @@ public void udfShouldNotOverrideInterfaceMethods() throws IOException { .withErrorCount(1) .withErrorContaining(Constants.MORE_THAN_ONE_TYPE_OVERRIDING_ERROR) .in(forResource("udfs/UDFOverridingInterfaceMethod.java")) - .onLine(14) + .onLine(13) .atColumn(8); } @@ -123,7 +123,7 @@ public void udfShouldImplementTopLevelStdUDF() throws IOException { .withErrorCount(1) .withErrorContaining(Constants.INTERFACE_NOT_IMPLEMENTED_ERROR) .in(forResource("udfs/UDFNotImplementingTopLevelStdUDF.java")) - .onLine(14) + .onLine(13) .atColumn(8); } diff --git a/transportable-udfs-annotation-processor/src/test/resources/outputs/empty.json b/transportable-udfs-annotation-processor/src/test/resources/outputs/empty.json index 7836d3b6..4e9cfcb3 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/outputs/empty.json +++ b/transportable-udfs-annotation-processor/src/test/resources/outputs/empty.json @@ -1,3 +1,4 @@ { - "udfs": [] + "udfs": {}, + "classToNumberOfTypeParameters": {} } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/outputs/overloadedUDF.json b/transportable-udfs-annotation-processor/src/test/resources/outputs/overloadedUDF.json index 9f6a2450..4f0526f0 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/outputs/overloadedUDF.json +++ b/transportable-udfs-annotation-processor/src/test/resources/outputs/overloadedUDF.json @@ -1,11 +1,13 @@ { - "udfs": [ - { - "topLevelClass": "udfs.OverloadedUDF1", - "stdUDFImplementations": [ - "udfs.OverloadedUDFInt", - "udfs.OverloadedUDFString" - ] - } - ] + "udfs": { + "udfs.OverloadedUDF1": [ + "udfs.OverloadedUDFInt", + "udfs.OverloadedUDFString" + ] + }, + "classToNumberOfTypeParameters": { + "udfs.OverloadedUDFString": 0, + "udfs.OverloadedUDF1": 0, + "udfs.OverloadedUDFInt": 0 + } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/outputs/simpleUDF.json b/transportable-udfs-annotation-processor/src/test/resources/outputs/simpleUDF.json index 9323ddbd..34c7cee2 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/outputs/simpleUDF.json +++ b/transportable-udfs-annotation-processor/src/test/resources/outputs/simpleUDF.json @@ -1,10 +1,10 @@ { - "udfs": [ - { - "topLevelClass": "udfs.SimpleUDF", - "stdUDFImplementations": [ - "udfs.SimpleUDF" - ] - } - ] + "udfs": { + "udfs.SimpleUDF": [ + "udfs.SimpleUDF" + ] + }, + "classToNumberOfTypeParameters": { + "udfs.SimpleUDF": 0 + } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDF.json b/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDF.json index ab58d4d8..5b72a274 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDF.json +++ b/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDF.json @@ -1,10 +1,10 @@ { - "udfs": [ - { - "topLevelClass": "udfs.UDFExtendingAbstractUDF", - "stdUDFImplementations": [ - "udfs.UDFExtendingAbstractUDF" - ] - } - ] + "udfs": { + "udfs.UDFExtendingAbstractUDF": [ + "udfs.UDFExtendingAbstractUDF" + ] + }, + "classToNumberOfTypeParameters": { + "udfs.UDFExtendingAbstractUDF": 0 + } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDFImplementingInterface.json b/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDFImplementingInterface.json index d2551a77..b75531e1 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDFImplementingInterface.json +++ b/transportable-udfs-annotation-processor/src/test/resources/outputs/udfExtendingAbstractUDFImplementingInterface.json @@ -1,10 +1,11 @@ { - "udfs": [ - { - "topLevelClass": "udfs.AbstractUDFImplementingInterface", - "stdUDFImplementations": [ - "udfs.UDFExtendingAbstractUDFImplementingInterface" - ] - } - ] + "udfs": { + "udfs.AbstractUDFImplementingInterface": [ + "udfs.UDFExtendingAbstractUDFImplementingInterface" + ] + }, + "classToNumberOfTypeParameters": { + "udfs.UDFExtendingAbstractUDFImplementingInterface": 0, + "udfs.AbstractUDFImplementingInterface": 0 + } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDF.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDF.java index 4a482115..06536aa2 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDF.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDF.java @@ -5,11 +5,10 @@ */ package udfs; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.TopLevelStdUDF; -public abstract class AbstractUDF extends StdUDF0 implements TopLevelStdUDF { +public abstract class AbstractUDF extends StdUDF0 implements TopLevelStdUDF { } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDFImplementingInterface.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDFImplementingInterface.java index 4078d7bc..7d85fb36 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDFImplementingInterface.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/AbstractUDFImplementingInterface.java @@ -5,12 +5,11 @@ */ package udfs; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.TopLevelStdUDF; -public abstract class AbstractUDFImplementingInterface extends StdUDF0 implements TopLevelStdUDF { +public abstract class AbstractUDFImplementingInterface extends StdUDF0 implements TopLevelStdUDF { @Override public String getFunctionName() { diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/OuterClassForInnerUDF.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/OuterClassForInnerUDF.java index d5d2551d..a84ee746 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/OuterClassForInnerUDF.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/OuterClassForInnerUDF.java @@ -6,14 +6,13 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; public class OuterClassForInnerUDF { - public class InnerUDF extends StdUDF0 implements TopLevelStdUDF { + public class InnerUDF extends StdUDF0 implements TopLevelStdUDF { @Override public String getFunctionName() { @@ -36,7 +35,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFInt.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFInt.java index 3f130d9d..292c8606 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFInt.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFInt.java @@ -6,12 +6,11 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdInteger; import com.linkedin.transport.api.udf.StdUDF0; import java.util.List; -public class OverloadedUDFInt extends StdUDF0 implements OverloadedUDF1 { +public class OverloadedUDFInt extends StdUDF0 implements OverloadedUDF1 { @Override public List getInputParameterSignatures() { @@ -24,7 +23,7 @@ public String getOutputParameterSignature() { } @Override - public StdInteger eval() { + public Integer eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFString.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFString.java index 9782d683..d0855f55 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFString.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/OverloadedUDFString.java @@ -6,12 +6,11 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import java.util.List; -public class OverloadedUDFString extends StdUDF0 implements OverloadedUDF1 { +public class OverloadedUDFString extends StdUDF0 implements OverloadedUDF1 { @Override public List getInputParameterSignatures() { @@ -24,7 +23,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/SimpleUDF.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/SimpleUDF.java index 15e749ec..46231c63 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/SimpleUDF.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/SimpleUDF.java @@ -6,13 +6,12 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class SimpleUDF extends StdUDF0 implements TopLevelStdUDF { +public class SimpleUDF extends StdUDF0 implements TopLevelStdUDF { @Override public String getFunctionName() { @@ -35,7 +34,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDF.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDF.java index 99fb068c..564fcd7e 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDF.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDF.java @@ -6,7 +6,6 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; @@ -34,7 +33,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDFImplementingInterface.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDFImplementingInterface.java index 2fc6d5ce..db8e3bc7 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDFImplementingInterface.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFExtendingAbstractUDFImplementingInterface.java @@ -6,7 +6,6 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import java.util.List; @@ -23,7 +22,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFNotImplementingTopLevelStdUDF.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFNotImplementingTopLevelStdUDF.java index 43e8ffa9..862403e4 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFNotImplementingTopLevelStdUDF.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFNotImplementingTopLevelStdUDF.java @@ -6,12 +6,11 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF0; import java.util.List; -public class UDFNotImplementingTopLevelStdUDF extends StdUDF0 { +public class UDFNotImplementingTopLevelStdUDF extends StdUDF0 { @Override public List getInputParameterSignatures() { @@ -24,7 +23,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFOverridingInterfaceMethod.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFOverridingInterfaceMethod.java index 346ff553..2d97547e 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFOverridingInterfaceMethod.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFOverridingInterfaceMethod.java @@ -6,12 +6,11 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdBoolean; import com.linkedin.transport.api.udf.StdUDF0; import java.util.List; -public class UDFOverridingInterfaceMethod extends StdUDF0 implements OverloadedUDF1 { +public class UDFOverridingInterfaceMethod extends StdUDF0 implements OverloadedUDF1 { @Override public String getFunctionName() { @@ -29,7 +28,7 @@ public String getOutputParameterSignature() { } @Override - public StdBoolean eval() { + public Boolean eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces1.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces1.java index 54e57c16..83a38c4d 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces1.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces1.java @@ -6,12 +6,11 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdBoolean; import com.linkedin.transport.api.udf.StdUDF0; import java.util.List; -public class UDFWithMultipleInterfaces1 extends StdUDF0 implements OverloadedUDF1, OverloadedUDF2 { +public class UDFWithMultipleInterfaces1 extends StdUDF0 implements OverloadedUDF1, OverloadedUDF2 { @Override public String getFunctionName() { @@ -34,7 +33,7 @@ public String getOutputParameterSignature() { } @Override - public StdBoolean eval() { + public Boolean eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces2.java b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces2.java index e8e62bb1..f2fbe270 100644 --- a/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces2.java +++ b/transportable-udfs-annotation-processor/src/test/resources/udfs/UDFWithMultipleInterfaces2.java @@ -6,7 +6,6 @@ package udfs; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdString; import java.util.List; @@ -33,7 +32,7 @@ public String getOutputParameterSignature() { } @Override - public StdString eval() { + public String eval() { return null; } } \ No newline at end of file diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/StdFactory.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/StdFactory.java index 3b9490af..d04d0ff5 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/StdFactory.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/StdFactory.java @@ -5,17 +5,11 @@ */ package com.linkedin.transport.api; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdBoolean; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdInteger; -import com.linkedin.transport.api.data.StdLong; -import com.linkedin.transport.api.data.StdMap; -import com.linkedin.transport.api.data.StdString; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdArrayType; import com.linkedin.transport.api.types.StdMapType; -import com.linkedin.transport.api.types.StdStructType; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF; import java.io.Serializable; @@ -23,7 +17,8 @@ /** - * {@link StdFactory} is used to create {@link StdData} and {@link StdType} objects inside Standard UDFs. + * {@link StdFactory} is used to create containter types (e.g., {@link ArrayData}, {@link MapData}, {@link RowData}) + * and {@link StdType} objects inside Standard UDFs. * * Specific APIs of {@link StdFactory} are implemented by each target platform (e.g., Spark, Presto, Hive) individually. * A {@link StdFactory} object is available inside Standard UDFs using {@link StdUDF#getStdFactory()}. @@ -32,111 +27,79 @@ public interface StdFactory extends Serializable { /** - * Creates a {@link StdInteger} representing a given integer value. - * - * @param value the input integer value - * @return {@link StdInteger} with the given integer value - */ - StdInteger createInteger(int value); - - /** - * Creates a {@link StdLong} representing a given long value. - * - * @param value the input long value - * @return {@link StdLong} with the given long value - */ - StdLong createLong(long value); - - /** - * Creates a {@link StdBoolean} representing a given boolean value. - * - * @param value the input boolean value - * @return {@link StdBoolean} with the given boolean value - */ - StdBoolean createBoolean(boolean value); - - /** - * Creates a {@link StdString} representing a given {@link String} value. - * - * @param value the input {@link String} value - * @return {@link StdString} with the given {@link String} value - */ - StdString createString(String value); - - /** - * Creates an empty {@link StdArray} whose type is given by the given {@link StdType}. + * Creates an empty {@link ArrayData} whose type is given by the given {@link StdType}. * * It is expected that the top-level {@link StdType} is a {@link StdArrayType}. * * @param stdType type of the array to be created * @param expectedSize expected number of entries in the array - * @return an empty {@link StdArray} + * @return an empty {@link ArrayData} */ - StdArray createArray(StdType stdType, int expectedSize); + ArrayData createArray(StdType stdType, int expectedSize); /** - * Creates an empty {@link StdArray} whose type is given by the given {@link StdType}. + * Creates an empty {@link ArrayData} whose type is given by the given {@link StdType}. * * It is expected that the top-level {@link StdType} is a {@link StdArrayType}. * * @param stdType type of the array to be created - * @return an empty {@link StdArray} + * @return an empty {@link ArrayData} */ - StdArray createArray(StdType stdType); + ArrayData createArray(StdType stdType); /** - * Creates an empty {@link StdMap} whose type is given by the given {@link StdType}. + * Creates an empty {@link MapData} whose type is given by the given {@link StdType}. * * It is expected that the top-level {@link StdType} is a {@link StdMapType}. * * @param stdType type of the map to be created - * @return an empty {@link StdMap} + * @return an empty {@link MapData} */ - StdMap createMap(StdType stdType); + MapData createMap(StdType stdType); /** - * Creates a {@link StdStruct} with the given field names and types. + * Creates a {@link RowData} with the given field names and types. * * @param fieldNames names of the struct fields * @param fieldTypes types of the struct fields - * @return a {@link StdStruct} with all fields initialized to null + * @return a {@link RowData} with all fields initialized to null */ - StdStruct createStruct(List fieldNames, List fieldTypes); + RowData createStruct(List fieldNames, List fieldTypes); /** - * Creates a {@link StdStruct} with the given field types. Field names will be field0, field1, field2... + * Creates a {@link RowData} with the given field types. Field names will be field0, field1, field2... * * @param fieldTypes types of the struct fields - * @return a {@link StdStruct} with all fields initialized to null + * @return a {@link RowData} with all fields initialized to null */ - StdStruct createStruct(List fieldTypes); + RowData createStruct(List fieldTypes); /** - * Creates a {@link StdStruct} whose type is given by the given {@link StdType}. + * Creates a {@link RowData} whose type is given by the given {@link StdType}. * - * It is expected that the top-level {@link StdType} is a {@link StdStructType}. + * It is expected that the top-level {@link StdType} is a {@link com.linkedin.transport.api.types.RowType}. * * @param stdType type of the struct to be created - * @return a {@link StdStruct} with all fields initialized to null + * @return a {@link RowData} with all fields initialized to null */ - StdStruct createStruct(StdType stdType); + RowData createStruct(StdType stdType); /** * Creates a {@link StdType} representing the given type signature. * * The following are considered valid type signatures: *
    - *
  • {@code "varchar"} - Represents SQL varchar type. Corresponding standard type is {@link StdString}
    • - *
  • {@code "integer"} - Represents SQL int type. Corresponding standard type is {@link StdInteger}
    • - *
  • {@code "bigint"} - Represents SQL bigint/long type. Corresponding standard type is {@link StdLong}
    • - *
  • {@code "boolean"} - Represents SQL boolean type. Corresponding standard type is {@link StdBoolean}
    • + *
  • {@code "varchar"} - Represents SQL varchar type. Corresponding Transport type is {@link String}
    • + *
  • {@code "integer"} - Represents SQL int type. Corresponding Transport type is {@link Integer}
    • + *
  • {@code "bigint"} - Represents SQL bigint/long type. Corresponding Transport type is {@link Long}
    • + *
  • {@code "boolean"} - Represents SQL boolean type. Corresponding Transport type is {@link Boolean}
    • *
  • {@code "array(T)"} - Represents SQL array type, where {@code T} is type signature of array element. - * Corresponding standard type is {@link StdArray}
    • + * Corresponding Transport type is {@link ArrayData} *
  • {@code "map(K,V)"} - Represents SQL map type, where {@code K} and {@code V} are type signatures of the map - * keys and values respectively. array element. Corresponding standard type is {@link StdMap}
    • + * keys and values respectively. Corresponding Transport type is {@link MapData} *
  • {@code "row(f0 T0, f1 T1,... fn Tn)"} - Represents SQL struct type, where {@code f0}...{@code fn} are field * names and {@code T0}...{@code Tn} are type signatures for the fields. Field names are optional; if not - * specified they default to {@code field0}...{@code fieldn}. Corresponding standard type is {@link StdStruct}
    • + * specified they default to {@code field0}...{@code fieldn}. Corresponding Transport type is {@link RowData} *
* * Generic type parameters can also be used as part of the type signatures; e.g., The type signature {@code "map(K,V)"} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdArray.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/ArrayData.java similarity index 76% rename from transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdArray.java rename to transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/ArrayData.java index ac698ae8..65a6b9a6 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdArray.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/ArrayData.java @@ -5,8 +5,8 @@ */ package com.linkedin.transport.api.data; -/** A Standard UDF data type for representing arrays. */ -public interface StdArray extends StdData, Iterable { +/** A Transport UDF data type for representing arrays. */ +public interface ArrayData extends Iterable { /** Returns the number of elements in the array. */ int size(); @@ -16,12 +16,12 @@ public interface StdArray extends StdData, Iterable { * * @param idx the index of the element to be retrieved */ - StdData get(int idx); + E get(int idx); /** * Adds an element to the end of the array. * * @param e the element to append to the array */ - void add(StdData e); + void add(E e); } diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdMap.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/MapData.java similarity index 78% rename from transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdMap.java rename to transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/MapData.java index 8e67500e..39bd6965 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdMap.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/MapData.java @@ -9,8 +9,8 @@ import java.util.Set; -/** A Standard UDF data type for representing maps. */ -public interface StdMap extends StdData { +/** A Transport UDF data type for representing maps. */ +public interface MapData { /** Returns the number of key-value pairs in the map. */ int size(); @@ -20,7 +20,7 @@ public interface StdMap extends StdData { * * @param key the key whose value is to be returned */ - StdData get(StdData key); + V get(K key); /** * Adds the given value to the map against the given key. @@ -28,18 +28,18 @@ public interface StdMap extends StdData { * @param key the key to which the value is to be associated * @param value the value to be associated with the key */ - void put(StdData key, StdData value); + void put(K key, V value); /** Returns a {@link Set} of all the keys in the map. */ - Set keySet(); + Set keySet(); /** Returns a {@link Collection} of all the values in the map. */ - Collection values(); + Collection values(); /** * Returns true if the map contains the given key, false otherwise. * * @param key the key to be checked */ - boolean containsKey(StdData key); + boolean containsKey(K key); } diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/PlatformData.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/PlatformData.java index 75df0518..41fc7bae 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/PlatformData.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/PlatformData.java @@ -5,7 +5,7 @@ */ package com.linkedin.transport.api.data; -/** An interface for all platform-specific implementations of {@link StdData}. */ +/** An interface to handle platform-specific container types. */ public interface PlatformData { /** Returns the underlying platform-specific object holding the data. */ diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdStruct.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/RowData.java similarity index 50% rename from transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdStruct.java rename to transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/RowData.java index 14ccff80..2d8f1ce0 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdStruct.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/RowData.java @@ -8,39 +8,39 @@ import java.util.List; -/** A Standard UDF data type for representing structs. */ -public interface StdStruct extends StdData { +/** A Transport UDF data type for representing SQL ROW/STRUCT data type. */ +public interface RowData { /** - * Returns the value of the field at the given position in the struct. + * Returns the value of the field at the given position in the row. * - * @param index the position of the field in the struct + * @param index the position of the field in the row */ - StdData getField(int index); + Object getField(int index); /** - * Returns the value of the given field from the struct. + * Returns the value of the given field from the row. * * @param name the name of the field */ - StdData getField(String name); + Object getField(String name); /** - * Sets the value of the field at the given position in the struct. + * Sets the value of the field at the given position in the row. * - * @param index the position of the field in the struct + * @param index the position of the field in the row * @param value the value to be set */ - void setField(int index, StdData value); + void setField(int index, Object value); /** - * Sets the value of the given field in the struct. + * Sets the value of the given field in the row. * * @param name the name of the field * @param value the value to be set */ - void setField(String name, StdData value); + void setField(String name, Object value); /** Returns a {@link List} of all fields in the struct. */ - List fields(); + List fields(); } diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdBoolean.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdBoolean.java deleted file mode 100644 index ff230bc1..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdBoolean.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -/** A Standard UDF data type for representing booleans. */ -public interface StdBoolean extends StdData { - - /** Returns the underlying boolean value. */ - boolean get(); -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdData.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdData.java deleted file mode 100644 index 77b3d1d7..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdData.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -import com.linkedin.transport.api.StdFactory; - - -/** - * An interface for all data types in Standard UDFs. - * - * {@link StdData} is the main interface through which StdUDFs receive input data and return output data. All Standard - * UDF data types (e.g., {@link StdInteger}, {@link StdArray}, {@link StdMap}) must extend {@link StdData}. Methods - * inside {@link StdFactory} can be used to create {@link StdData} objects. - */ -public interface StdData { -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdInteger.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdInteger.java deleted file mode 100644 index c74a92dd..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdInteger.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -/** A Standard UDF data type for representing integers. */ -public interface StdInteger extends StdData { - - /** Returns the underlying int value. */ - int get(); -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdLong.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdLong.java deleted file mode 100644 index 84f322f7..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdLong.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -/** A Standard UDF data type for representing longs. */ -public interface StdLong extends StdData { - - /** Returns the underlying long value. */ - long get(); -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdString.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdString.java deleted file mode 100644 index 7ccd8385..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdString.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -/** A Standard UDF data type for representing strings. */ -public interface StdString extends StdData { - - /** Returns the underlying {@link String} value. */ - String get(); -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdTimestamp.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdTimestamp.java deleted file mode 100644 index 18ff9bc1..00000000 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/data/StdTimestamp.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.api.data; - -/** A Standard UDF data type for representing timestamps. */ -public interface StdTimestamp extends StdData { - - /** Returns the number of milliseconds elapsed from epoch for the {@link StdTimestamp}. */ - long toEpoch(); -} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdStructType.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/RowType.java similarity index 89% rename from transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdStructType.java rename to transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/RowType.java index 521ec2d7..59938208 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdStructType.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/RowType.java @@ -9,7 +9,7 @@ /** A {@link StdType} representing a struct type. */ -public interface StdStructType extends StdType { +public interface RowType extends StdType { /** Returns a {@link List} of the types of all the struct fields. */ List fieldTypes(); diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdBinaryType.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdBinaryType.java new file mode 100644 index 00000000..5fbe53e1 --- /dev/null +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdBinaryType.java @@ -0,0 +1,10 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.api.types; + +/** A {@link StdType} representing a {@link java.nio.ByteBuffer} type. */ +public interface StdBinaryType extends StdType { +} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdDoubleType.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdDoubleType.java new file mode 100644 index 00000000..1179729a --- /dev/null +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdDoubleType.java @@ -0,0 +1,10 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.api.types; + +/** A {@link StdType} representing a double type. */ +public interface StdDoubleType extends StdType { +} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdFloatType.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdFloatType.java new file mode 100644 index 00000000..d1ff9952 --- /dev/null +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/types/StdFloatType.java @@ -0,0 +1,10 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.api.types; + +/** A {@link StdType} representing a float type. */ +public interface StdFloatType extends StdType { +} diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF.java index a4e29220..f90b91ca 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF.java @@ -6,8 +6,6 @@ package com.linkedin.transport.api.udf; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.types.StdType; import java.util.List; @@ -19,8 +17,7 @@ * abstract class for UDFs expecting {@code i} arguments. Similar to lambda expressions, StdUDF(i) abstract classes are * type-parameterized by the input types and output type of the eval function. Each class is type-parameterized by * {@code (i+1)} type parameters; {@code i} type parameters for the UDF input types, and one type parameter for the - * output type. All types (both input and output types) must extend the {@link StdData} - * interface. + * output type. */ public abstract class StdUDF { private StdFactory _stdFactory; @@ -40,7 +37,7 @@ public abstract class StdUDF { * of contained UDF. * * @param stdFactory a {@link StdFactory} object which can be used to create - * {@link StdData} and {@link StdType} objects + * data and type objects */ public void init(StdFactory stdFactory) { _stdFactory = stdFactory; @@ -85,8 +82,8 @@ public final boolean[] getAndCheckNullableArguments() { protected abstract int numberOfArguments(); /** - * Returns a {@link StdFactory} object which can be used to create {@link StdData} and - * {@link StdType} objects + * Returns a {@link StdFactory} object which can be used to create data and + * type objects */ public StdFactory getStdFactory() { return _stdFactory; diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF0.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF0.java index b62fe95f..d3558fc3 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF0.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF0.java @@ -5,15 +5,13 @@ */ package com.linkedin.transport.api.udf; -import com.linkedin.transport.api.data.StdData; - /** * A Standard UDF with zero input arguments. * * @param the type of the return value of the {@link StdUDF} */ -public abstract class StdUDF0 extends StdUDF { +public abstract class StdUDF0 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF1.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF1.java index 28d0ad71..18e8e769 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF1.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF1.java @@ -5,8 +5,6 @@ */ package com.linkedin.transport.api.udf; -import com.linkedin.transport.api.data.StdData; - /** * A Standard UDF with one input argument. @@ -17,7 +15,7 @@ // Suppressing class parameter type parameter name and arg naming style checks since this naming convention is more // suitable to Standard UDFs, and the code is more readable this way. @SuppressWarnings({"checkstyle:classtypeparametername", "checkstyle:regexpsinglelinejava"}) -public abstract class StdUDF1 extends StdUDF { +public abstract class StdUDF1 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -38,7 +36,7 @@ public abstract class StdUDF1 extends Std * hence obtaining the most recent version of a file. * Example: 'hdfs:///data/derived/dwh/prop/testMemberId/#LATEST/testMemberId.txt' * - * The arguments passed to {@link #eval(StdData)} are passed to this method as well to allow users to construct + * The arguments passed to {@link #eval(Object)} are passed to this method as well to allow users to construct * required file paths from arguments passed to the UDF. Since this method is called before any rows are processed, * only constant UDF arguments should be used to construct the file paths. Values of non-constant arguments are not * deterministic, and are null for most platforms. (Constant arguments are arguments whose literal values are given diff --git a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF2.java b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF2.java index 3e020ae1..3eb293ba 100644 --- a/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF2.java +++ b/transportable-udfs-api/src/main/java/com/linkedin/transport/api/udf/StdUDF2.java @@ -5,8 +5,6 @@ */ package com.linkedin.transport.api.udf; -import com.linkedin.transport.api.data.StdData; - /** * A Standard UDF with three input arguments. @@ -18,7 +16,7 @@ // Suppressing class parameter type parameter name and arg naming style checks since this naming convention is more // suitable to Standard UDFs, and the code is more readable this way. @SuppressWarnings({"checkstyle:classtypeparametername", "checkstyle:regexpsinglelinejava"}) -public abstract class StdUDF2 extends StdUDF { +public abstract class StdUDF2 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -40,7 +38,7 @@ public abstract class StdUDF2 +public abstract class StdUDF3 extends StdUDF { /** @@ -43,7 +41,7 @@ public abstract class StdUDF3 +public abstract class StdUDF4 extends StdUDF { /** @@ -45,7 +43,7 @@ public abstract class StdUDF4 extends StdUDF { +public abstract class StdUDF5 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -47,7 +45,7 @@ public abstract class StdUDF5 extends StdUDF { +public abstract class StdUDF6 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -49,7 +47,7 @@ public abstract class StdUDF6 extends StdUDF { +public abstract class StdUDF7 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -51,7 +49,7 @@ public abstract class StdUDF7 extends StdUDF { +public abstract class StdUDF8 extends StdUDF { /** * Returns the output of the {@link StdUDF} given the input arguments. @@ -53,7 +51,7 @@ public abstract class StdUDF8 boundVariables) { } @Override - public StdInteger createInteger(int value) { - return new AvroInteger(value); + public ArrayData createArray(StdType stdType, int size) { + return new AvroArrayData((Schema) stdType.underlyingType(), size); } @Override - public StdLong createLong(long value) { - return new AvroLong(value); - } - - @Override - public StdBoolean createBoolean(boolean value) { - return new AvroBoolean(value); - } - - @Override - public StdString createString(String value) { - return new AvroString(new Utf8(value)); - } - - @Override - public StdArray createArray(StdType stdType, int size) { - return new AvroArray((Schema) stdType.underlyingType(), size); - } - - @Override - public StdArray createArray(StdType stdType) { + public ArrayData createArray(StdType stdType) { return createArray(stdType, 0); } @Override - public StdMap createMap(StdType stdType) { - return new AvroMap((Schema) stdType.underlyingType()); + public MapData createMap(StdType stdType) { + return new AvroMapData((Schema) stdType.underlyingType()); } @Override - public StdStruct createStruct(List fieldNames, List fieldTypes) { + public RowData createStruct(List fieldNames, List fieldTypes) { if (fieldNames.size() != fieldTypes.size()) { throw new RuntimeException( "Field names and types are of different lengths: " + "Field names length is " + fieldNames.size() + ". " @@ -90,18 +62,18 @@ public StdStruct createStruct(List fieldNames, List fieldTypes) for (int i = 0; i < fieldTypes.size(); i++) { fields.add(new Field(fieldNames.get(i), (Schema) fieldTypes.get(i).underlyingType(), null, null)); } - return new AvroStruct(Schema.createRecord(fields)); + return new AvroRowData(Schema.createRecord(fields)); } @Override - public StdStruct createStruct(List fieldTypes) { + public RowData createStruct(List fieldTypes) { return createStruct(IntStream.range(0, fieldTypes.size()).mapToObj(i -> "field" + i).collect(Collectors.toList()), fieldTypes); } @Override - public StdStruct createStruct(StdType stdType) { - return new AvroStruct((Schema) stdType.underlyingType()); + public RowData createStruct(StdType stdType) { + return new AvroRowData((Schema) stdType.underlyingType()); } @Override diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/AvroWrapper.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/AvroWrapper.java index a4c65904..b493f680 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/AvroWrapper.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/AvroWrapper.java @@ -5,22 +5,23 @@ */ package com.linkedin.transport.avro; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.PlatformData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.avro.data.AvroArray; -import com.linkedin.transport.avro.data.AvroBoolean; -import com.linkedin.transport.avro.data.AvroInteger; -import com.linkedin.transport.avro.data.AvroLong; -import com.linkedin.transport.avro.data.AvroMap; -import com.linkedin.transport.avro.data.AvroString; -import com.linkedin.transport.avro.data.AvroStruct; +import com.linkedin.transport.avro.data.AvroArrayData; +import com.linkedin.transport.avro.data.AvroMapData; +import com.linkedin.transport.avro.data.AvroRowData; import com.linkedin.transport.avro.types.AvroArrayType; +import com.linkedin.transport.avro.types.AvroBinaryType; import com.linkedin.transport.avro.types.AvroBooleanType; +import com.linkedin.transport.avro.types.AvroDoubleType; +import com.linkedin.transport.avro.types.AvroFloatType; import com.linkedin.transport.avro.types.AvroIntegerType; import com.linkedin.transport.avro.types.AvroLongType; import com.linkedin.transport.avro.types.AvroMapType; import com.linkedin.transport.avro.types.AvroStringType; -import com.linkedin.transport.avro.types.AvroStructType; +import com.linkedin.transport.avro.types.AvroRowType; +import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; import org.apache.avro.Schema; import org.apache.avro.generic.GenericArray; @@ -33,22 +34,35 @@ public class AvroWrapper { private AvroWrapper() { } - public static StdData createStdData(Object avroData, Schema avroSchema) { + public static Object createStdData(Object avroData, Schema avroSchema) { switch (avroSchema.getType()) { case INT: - return new AvroInteger((Integer) avroData); case LONG: - return new AvroLong((Long) avroData); case BOOLEAN: - return new AvroBoolean((Boolean) avroData); + case FLOAT: + case DOUBLE: + case BYTES: + return avroData; case STRING: - return new AvroString((Utf8) avroData); + case ENUM: + if (avroData == null) { + return null; + } else { + return avroData.toString(); + } case ARRAY: - return new AvroArray((GenericArray) avroData, avroSchema); + return new AvroArrayData((GenericArray) avroData, avroSchema); case MAP: - return new AvroMap((Map) avroData, avroSchema); + return new AvroMapData((Map) avroData, avroSchema); case RECORD: - return new AvroStruct((GenericRecord) avroData, avroSchema); + return new AvroRowData((GenericRecord) avroData, avroSchema); + case UNION: { + Schema nonNullableType = getNonNullComponent(avroSchema); + if (avroData == null) { + return null; + } + return createStdData(avroData, nonNullableType); + } case NULL: return null; default: @@ -56,6 +70,36 @@ public static StdData createStdData(Object avroData, Schema avroSchema) { } } + public static Object getPlatformData(Object transportData) { + if (transportData instanceof Integer || transportData instanceof Long || transportData instanceof Double || + transportData instanceof Boolean || transportData instanceof ByteBuffer) { + return transportData; + } else if (transportData instanceof String) { + return transportData == null? null : new Utf8((String) transportData); + } else { + return transportData == null ? null : ((PlatformData) transportData).getUnderlyingData(); + } + } + + + /** + * Returns a non null component of a simple union schema. The supported union schema must have + * only two fields where one of them is null type, the other is returned. + */ + private static Schema getNonNullComponent(Schema unionSchema) { + List types = unionSchema.getTypes(); + if (types.size() == 2) { + if (types.get(0).getType().equals(Schema.Type.NULL)) { + return types.get(1); + } + + if (types.get(1).getType().equals(Schema.Type.NULL)) { + return types.get(0); + } + } + throw new RuntimeException("Unsupported union type: " + unionSchema); + } + public static StdType createStdType(Schema avroSchema) { switch (avroSchema.getType()) { case INT: @@ -66,12 +110,22 @@ public static StdType createStdType(Schema avroSchema) { return new AvroBooleanType(avroSchema); case STRING: return new AvroStringType(avroSchema); + case FLOAT: + return new AvroFloatType(avroSchema); + case DOUBLE: + return new AvroDoubleType(avroSchema); + case BYTES: + return new AvroBinaryType(avroSchema); case ARRAY: return new AvroArrayType(avroSchema); case MAP: return new AvroMapType(avroSchema); case RECORD: - return new AvroStructType(avroSchema); + return new AvroRowType(avroSchema); + case UNION: { + Schema nonNullableType = getNonNullComponent(avroSchema); + return createStdType(nonNullableType); + } default: throw new RuntimeException("Unrecognized Avro Schema: " + avroSchema.getClass()); } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/StdUdfWrapper.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/StdUdfWrapper.java index a1ea3e0d..41eb59b4 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/StdUdfWrapper.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/StdUdfWrapper.java @@ -7,7 +7,6 @@ import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.StdUDF1; @@ -36,7 +35,7 @@ public abstract class StdUdfWrapper { protected boolean _requiredFilesProcessed; protected StdFactory _stdFactory; private boolean[] _nullableArguments; - private StdData[] _args; + private Object[] _args; /** * Given input schemas, this method matches them to the expected type signatures, and finds bindings to the @@ -68,12 +67,27 @@ protected boolean containsNullValuedNonNullableArgument(Object[] arguments) { return false; } - protected StdData wrap(Object avroObject, StdData stdData) { - if (avroObject != null) { - ((PlatformData) stdData).setUnderlyingData(avroObject); - return stdData; - } else { - return null; + protected Object wrap(Object avroObject, Schema inputSchema, Object stdData) { + switch (inputSchema.getType()) { + case INT: + case LONG: + case BOOLEAN: + return avroObject; + case STRING: + return avroObject == null? null : avroObject.toString(); + case ARRAY: + case MAP: + case RECORD: + if (avroObject != null) { + ((PlatformData) stdData).setUnderlyingData(avroObject); + return stdData; + } else { + return null; + } + case NULL: + return null; + default: + throw new RuntimeException("Unrecognized Avro Schema: " + inputSchema.getClass()); } } @@ -82,22 +96,24 @@ protected StdData wrap(Object avroObject, StdData stdData) { protected abstract Class getTopLevelUdfClass(); protected void createStdData() { - _args = new StdData[_inputSchemas.length]; + _args = new Object[_inputSchemas.length]; for (int i = 0; i < _inputSchemas.length; i++) { _args[i] = AvroWrapper.createStdData(null, _inputSchemas[i]); } } - private StdData[] wrapArguments(Object[] arguments) { - return IntStream.range(0, _args.length).mapToObj(i -> wrap(arguments[i], _args[i])).toArray(StdData[]::new); + private Object[] wrapArguments(Object[] arguments) { + return IntStream.range(0, _args.length).mapToObj( + i -> wrap(arguments[i], _inputSchemas[i], _args[i]) + ).toArray(Object[]::new); } public Object evaluate(Object[] arguments) { if (containsNullValuedNonNullableArgument(arguments)) { return null; } - StdData[] args = wrapArguments(arguments); - StdData result; + Object[] args = wrapArguments(arguments); + Object result; switch (args.length) { case 0: result = ((StdUDF0) _stdUdf).eval(); @@ -129,6 +145,6 @@ public Object evaluate(Object[] arguments) { default: throw new UnsupportedOperationException("eval not yet supported for StdUDF" + args.length); } - return result == null ? null : ((PlatformData) result).getUnderlyingData(); + return AvroWrapper.getPlatformData(result); } } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArray.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArrayData.java similarity index 65% rename from transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArray.java rename to transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArrayData.java index 1557ed6c..3cb2e6f6 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArray.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroArrayData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.avro.data; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.avro.AvroWrapper; import java.util.Iterator; import org.apache.avro.Schema; @@ -15,16 +14,16 @@ import org.apache.avro.generic.GenericData; -public class AvroArray implements StdArray, PlatformData { +public class AvroArrayData implements ArrayData, PlatformData { private final Schema _elementSchema; private GenericArray _genericArray; - public AvroArray(GenericArray genericArray, Schema arraySchema) { + public AvroArrayData(GenericArray genericArray, Schema arraySchema) { _genericArray = genericArray; _elementSchema = arraySchema.getElementType(); } - public AvroArray(Schema arraySchema, int size) { + public AvroArrayData(Schema arraySchema, int size) { _elementSchema = arraySchema.getElementType(); _genericArray = new GenericData.Array(size, arraySchema); } @@ -35,18 +34,18 @@ public int size() { } @Override - public StdData get(int idx) { - return AvroWrapper.createStdData(_genericArray.get(idx), _elementSchema); + public E get(int idx) { + return (E) AvroWrapper.createStdData(_genericArray.get(idx), _elementSchema); } @Override - public void add(StdData e) { - _genericArray.add(((PlatformData) e).getUnderlyingData()); + public void add(E e) { + _genericArray.add(AvroWrapper.getPlatformData(e)); } @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { private final Iterator _iterator = _genericArray.iterator(); @Override @@ -55,8 +54,8 @@ public boolean hasNext() { } @Override - public StdData next() { - return AvroWrapper.createStdData(_iterator.next(), _elementSchema); + public E next() { + return (E) AvroWrapper.createStdData(_iterator.next(), _elementSchema); } }; } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroBoolean.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroBoolean.java deleted file mode 100644 index 99f83738..00000000 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroBoolean.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.avro.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdBoolean; - - -public class AvroBoolean implements StdBoolean, PlatformData { - private Boolean _boolean; - - public AvroBoolean(Boolean aBoolean) { - _boolean = aBoolean; - } - - @Override - public boolean get() { - return _boolean; - } - - @Override - public Object getUnderlyingData() { - return _boolean; - } - - @Override - public void setUnderlyingData(Object value) { - _boolean = (Boolean) value; - } -} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroInteger.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroInteger.java deleted file mode 100644 index 5a170f3b..00000000 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroInteger.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.avro.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdInteger; - - -public class AvroInteger implements StdInteger, PlatformData { - private Integer _integer; - - public AvroInteger(Integer integer) { - _integer = integer; - } - - @Override - public int get() { - return _integer; - } - - @Override - public Object getUnderlyingData() { - return _integer; - } - - @Override - public void setUnderlyingData(Object value) { - _integer = (Integer) value; - } -} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroLong.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroLong.java deleted file mode 100644 index a56af06c..00000000 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroLong.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.avro.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdLong; - - -public class AvroLong implements StdLong, PlatformData { - private Long _long; - - public AvroLong(Long aLong) { - _long = aLong; - } - - @Override - public long get() { - return _long; - } - - @Override - public Object getUnderlyingData() { - return _long; - } - - @Override - public void setUnderlyingData(Object value) { - _long = (Long) value; - } -} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMap.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMapData.java similarity index 59% rename from transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMap.java rename to transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMapData.java index d0913d53..2b95796c 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMap.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroMapData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.avro.data; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.avro.AvroWrapper; import java.util.AbstractSet; import java.util.Collection; @@ -21,18 +20,18 @@ import static org.apache.avro.Schema.Type.*; -public class AvroMap implements StdMap, PlatformData { +public class AvroMapData implements MapData, PlatformData { private Map _map; private final Schema _keySchema; private final Schema _valueSchema; - public AvroMap(Map map, Schema mapSchema) { + public AvroMapData(Map map, Schema mapSchema) { _map = map; _keySchema = Schema.create(STRING); _valueSchema = mapSchema.getValueType(); } - public AvroMap(Schema mapSchema) { + public AvroMapData(Schema mapSchema) { _map = new LinkedHashMap<>(); _keySchema = Schema.create(STRING); _valueSchema = mapSchema.getValueType(); @@ -54,21 +53,21 @@ public int size() { } @Override - public StdData get(StdData key) { - return AvroWrapper.createStdData(_map.get(((PlatformData) key).getUnderlyingData()), _valueSchema); + public V get(K key) { + return (V) AvroWrapper.createStdData(_map.get(AvroWrapper.getPlatformData(key)), _valueSchema); } @Override - public void put(StdData key, StdData value) { - _map.put(((PlatformData) key).getUnderlyingData(), ((PlatformData) value).getUnderlyingData()); + public void put(K key, V value) { + _map.put(AvroWrapper.getPlatformData(key), AvroWrapper.getPlatformData(value)); } @Override - public Set keySet() { - return new AbstractSet() { + public Set keySet() { + return new AbstractSet() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator keySet = _map.keySet().iterator(); @Override public boolean hasNext() { @@ -76,8 +75,8 @@ public boolean hasNext() { } @Override - public StdData next() { - return AvroWrapper.createStdData(keySet.next(), _keySchema); + public K next() { + return (K) AvroWrapper.createStdData(keySet.next(), _keySchema); } }; } @@ -90,12 +89,12 @@ public int size() { } @Override - public Collection values() { - return _map.values().stream().map(v -> AvroWrapper.createStdData(v, _valueSchema)).collect(Collectors.toList()); + public Collection values() { + return _map.values().stream().map(v -> (V) AvroWrapper.createStdData(v, _valueSchema)).collect(Collectors.toList()); } @Override - public boolean containsKey(StdData key) { - return _map.containsKey(((PlatformData) key).getUnderlyingData()); + public boolean containsKey(K key) { + return _map.containsKey(AvroWrapper.getPlatformData(key)); } } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroStruct.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroRowData.java similarity index 68% rename from transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroStruct.java rename to transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroRowData.java index f018d5bc..64fa5e4c 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroStruct.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroRowData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.avro.data; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.avro.AvroWrapper; import java.util.List; import java.util.stream.Collectors; @@ -17,17 +16,17 @@ import org.apache.avro.generic.GenericRecord; -public class AvroStruct implements StdStruct, PlatformData { +public class AvroRowData implements RowData, PlatformData { private final Schema _recordSchema; private GenericRecord _genericRecord; - public AvroStruct(GenericRecord genericRecord, Schema recordSchema) { + public AvroRowData(GenericRecord genericRecord, Schema recordSchema) { _genericRecord = genericRecord; _recordSchema = recordSchema; } - public AvroStruct(Schema recordSchema) { + public AvroRowData(Schema recordSchema) { _genericRecord = new Record(recordSchema); _recordSchema = recordSchema; } @@ -43,27 +42,27 @@ public void setUnderlyingData(Object value) { } @Override - public StdData getField(int index) { + public Object getField(int index) { return AvroWrapper.createStdData(_genericRecord.get(index), _recordSchema.getFields().get(index).schema()); } @Override - public StdData getField(String name) { + public Object getField(String name) { return AvroWrapper.createStdData(_genericRecord.get(name), _recordSchema.getField(name).schema()); } @Override - public void setField(int index, StdData value) { - _genericRecord.put(index, ((PlatformData) value).getUnderlyingData()); + public void setField(int index, Object value) { + _genericRecord.put(index, AvroWrapper.getPlatformData(value)); } @Override - public void setField(String name, StdData value) { - _genericRecord.put(name, ((PlatformData) value).getUnderlyingData()); + public void setField(String name, Object value) { + _genericRecord.put(name, AvroWrapper.getPlatformData(value)); } @Override - public List fields() { + public List fields() { return IntStream.range(0, _recordSchema.getFields().size()).mapToObj(i -> getField(i)).collect(Collectors.toList()); } } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroString.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroString.java deleted file mode 100644 index 745df05e..00000000 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/data/AvroString.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.avro.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdString; -import org.apache.avro.util.Utf8; - - -public class AvroString implements StdString, PlatformData { - private Utf8 _string; - - public AvroString(Utf8 string) { - _string = string; - } - - @Override - public String get() { - return _string.toString(); - } - - @Override - public Object getUnderlyingData() { - return _string; - } - - @Override - public void setUnderlyingData(Object value) { - _string = (Utf8) value; - } -} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroBinaryType.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroBinaryType.java new file mode 100644 index 00000000..883d37bc --- /dev/null +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroBinaryType.java @@ -0,0 +1,23 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.avro.types; + +import com.linkedin.transport.api.types.StdBinaryType; +import org.apache.avro.Schema; + + +public class AvroBinaryType implements StdBinaryType { + final private Schema _schema; + + public AvroBinaryType(Schema schema) { + _schema = schema; + } + + @Override + public Object underlyingType() { + return _schema; + } +} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroDoubleType.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroDoubleType.java new file mode 100644 index 00000000..fe9b847d --- /dev/null +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroDoubleType.java @@ -0,0 +1,23 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.avro.types; + +import com.linkedin.transport.api.types.StdDoubleType; +import org.apache.avro.Schema; + + +public class AvroDoubleType implements StdDoubleType { + final private Schema _schema; + + public AvroDoubleType(Schema schema) { + _schema = schema; + } + + @Override + public Object underlyingType() { + return _schema; + } +} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroFloatType.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroFloatType.java new file mode 100644 index 00000000..c277fd54 --- /dev/null +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroFloatType.java @@ -0,0 +1,23 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.avro.types; + +import com.linkedin.transport.api.types.StdFloatType; +import org.apache.avro.Schema; + + +public class AvroFloatType implements StdFloatType { + final private Schema _schema; + + public AvroFloatType(Schema schema) { + _schema = schema; + } + + @Override + public Object underlyingType() { + return _schema; + } +} diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroStructType.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroRowType.java similarity index 82% rename from transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroStructType.java rename to transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroRowType.java index 2c97b39c..3923b3f5 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroStructType.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/types/AvroRowType.java @@ -5,7 +5,7 @@ */ package com.linkedin.transport.avro.types; -import com.linkedin.transport.api.types.StdStructType; +import com.linkedin.transport.api.types.RowType; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.avro.AvroWrapper; import java.util.List; @@ -13,10 +13,10 @@ import org.apache.avro.Schema; -public class AvroStructType implements StdStructType { +public class AvroRowType implements RowType { final private Schema _schema; - public AvroStructType(Schema schema) { + public AvroRowType(Schema schema) { _schema = schema; } diff --git a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/typesystem/AvroTypeSystem.java b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/typesystem/AvroTypeSystem.java index 881fa709..f85e75d7 100644 --- a/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/typesystem/AvroTypeSystem.java +++ b/transportable-udfs-avro/src/main/java/com/linkedin/transport/avro/typesystem/AvroTypeSystem.java @@ -60,6 +60,21 @@ protected boolean isStringType(Schema dataType) { return dataType.getType() == STRING; } + @Override + protected boolean isFloatType(Schema dataType) { + return dataType.getType() == FLOAT; + } + + @Override + protected boolean isDoubleType(Schema dataType) { + return dataType.getType() == DOUBLE; + } + + @Override + protected boolean isBinaryType(Schema dataType) { + return dataType.getType() == BYTES; + } + @Override protected boolean isArrayType(Schema dataType) { return dataType.getType() == ARRAY; @@ -95,6 +110,21 @@ protected Schema createStringType() { return Schema.create(STRING); } + @Override + protected Schema createFloatType() { + return Schema.create(FLOAT); + } + + @Override + protected Schema createDoubleType() { + return Schema.create(DOUBLE); + } + + @Override + protected Schema createBinaryType() { + return Schema.create(BYTES); + } + @Override protected Schema createUnknownType() { return Schema.create(NULL); diff --git a/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/TestAvroWrapper.java b/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/TestAvroWrapper.java new file mode 100644 index 00000000..d81a452a --- /dev/null +++ b/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/TestAvroWrapper.java @@ -0,0 +1,264 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.avro; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.transport.api.data.PlatformData; +import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.types.StdType; +import com.linkedin.transport.avro.data.AvroArray; +import com.linkedin.transport.avro.data.AvroBinary; +import com.linkedin.transport.avro.data.AvroBoolean; +import com.linkedin.transport.avro.data.AvroDouble; +import com.linkedin.transport.avro.data.AvroFloat; +import com.linkedin.transport.avro.data.AvroInteger; +import com.linkedin.transport.avro.data.AvroLong; +import com.linkedin.transport.avro.data.AvroMap; +import com.linkedin.transport.avro.data.AvroString; +import com.linkedin.transport.avro.data.AvroStruct; +import com.linkedin.transport.avro.types.AvroArrayType; +import com.linkedin.transport.avro.types.AvroBinaryType; +import com.linkedin.transport.avro.types.AvroBooleanType; +import com.linkedin.transport.avro.types.AvroDoubleType; +import com.linkedin.transport.avro.types.AvroFloatType; +import com.linkedin.transport.avro.types.AvroIntegerType; +import com.linkedin.transport.avro.types.AvroLongType; +import com.linkedin.transport.avro.types.AvroMapType; +import com.linkedin.transport.avro.types.AvroStringType; +import com.linkedin.transport.avro.types.AvroStructType; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + + +public class TestAvroWrapper { + + private Schema createSchema(String typeName) { + return createSchema("testField", typeName); + } + + private Schema createSchema(String fieldName, String typeName) { + return new Schema.Parser().parse( + String.format("{\"name\": \"%s\",\"type\": %s}", fieldName, typeName)); + } + + private void testSimpleType(String typeName, Class expectedAvroTypeClass, + Object testData, Class expectedDataClass) { + Schema avroSchema = createSchema(String.format("\"%s\"", typeName)); + + StdType stdType = AvroWrapper.createStdType(avroSchema); + assertTrue(expectedAvroTypeClass.isAssignableFrom(stdType.getClass())); + assertEquals(avroSchema, stdType.underlyingType()); + + StdData stdData = AvroWrapper.createStdData(testData, avroSchema); + assertNotNull(stdData); + assertTrue(expectedDataClass.isAssignableFrom(stdData.getClass())); + if ("string".equals(typeName)) { + // Use String values for equality assertion as we support both Utf8 and String input types + assertEquals(testData.toString(), ((PlatformData) stdData).getUnderlyingData().toString()); + } else { + assertEquals(testData, ((PlatformData) stdData).getUnderlyingData()); + } + } + + @Test + public void testBooleanType() { + testSimpleType("boolean", AvroBooleanType.class, true, AvroBoolean.class); + } + + @Test + public void testIntegerType() { + testSimpleType("int", AvroIntegerType.class, 1, AvroInteger.class); + } + + @Test + public void testLongType() { + testSimpleType("long", AvroLongType.class, 1L, AvroLong.class); + } + + @Test + public void testFloatType() { + testSimpleType("float", AvroFloatType.class, 1.0f, AvroFloat.class); + } + + @Test + public void testDoubleType() { + testSimpleType("double", AvroDoubleType.class, 1.0, AvroDouble.class); + } + + @Test + public void testStringType() { + testSimpleType("string", AvroStringType.class, new Utf8("foo"), AvroString.class); + testSimpleType("string", AvroStringType.class, "foo", AvroString.class); + } + + @Test + public void testBinaryType() { + testSimpleType("bytes", AvroBinaryType.class, ByteBuffer.wrap("bar".getBytes()), AvroBinary.class); + } + + @Test + public void testEnumType() { + Schema field1 = createSchema("field1", "" + + "\"enum\"," + + "\"name\":\"SampleEnum\"," + + "\"doc\":\"\"," + + "\"symbols\":[\"A\",\"B\"]"); + Schema structSchema = Schema.createRecord(ImmutableList.of( + new Schema.Field("field1", field1, null, null) + )); + + GenericRecord record1 = new GenericData.Record(structSchema); + record1.put("field1", "A"); + StdData stdEnumData1 = AvroWrapper.createStdData(record1.get("field1"), + Schema.createEnum("SampleEnum", "", "", Arrays.asList("A", "B"))); + assertTrue(stdEnumData1 instanceof AvroString); + assertEquals("A", ((AvroString) stdEnumData1).get()); + + GenericRecord record2 = new GenericData.Record(structSchema); + record1.put("field1", new GenericData.EnumSymbol(field1, "A")); + StdData stdEnumData2 = AvroWrapper.createStdData(record1.get("field1"), + Schema.createEnum("SampleEnum", "", "", Arrays.asList("A", "B"))); + assertTrue(stdEnumData2 instanceof AvroString); + assertEquals("A", ((AvroString) stdEnumData2).get()); + } + + @Test + public void testArrayType() { + Schema elementType = createSchema("\"int\""); + Schema arraySchema = Schema.createArray(elementType); + + StdType stdArrayType = AvroWrapper.createStdType(arraySchema); + assertTrue(stdArrayType instanceof AvroArrayType); + assertEquals(arraySchema, stdArrayType.underlyingType()); + assertEquals(elementType, ((AvroArrayType) stdArrayType).elementType().underlyingType()); + + GenericArray value = new GenericData.Array<>(arraySchema, Arrays.asList(1, 2)); + StdData stdArrayData = AvroWrapper.createStdData(value, arraySchema); + assertTrue(stdArrayData instanceof AvroArray); + assertEquals(2, ((AvroArray) stdArrayData).size()); + assertEquals(value, ((AvroArray) stdArrayData).getUnderlyingData()); + } + + @Test + public void testMapType() { + Schema valueType = createSchema("\"long\""); + Schema mapSchema = Schema.createMap(valueType); + + StdType stdMapType = AvroWrapper.createStdType(mapSchema); + assertTrue(stdMapType instanceof AvroMapType); + assertEquals(mapSchema, stdMapType.underlyingType()); + assertEquals(valueType, ((AvroMapType) stdMapType).valueType().underlyingType()); + + Map value = ImmutableMap.of("foo", 1L, "bar", 2L); + StdData stdMapData = AvroWrapper.createStdData(value, mapSchema); + assertTrue(stdMapData instanceof AvroMap); + assertEquals(2, ((AvroMap) stdMapData).size()); + assertEquals(value, ((AvroMap) stdMapData).getUnderlyingData()); + } + + @Test + public void testRecordType() { + Schema field1 = createSchema("field1", "\"int\""); + Schema field2 = createSchema("field2", "\"double\""); + Schema structSchema = Schema.createRecord(ImmutableList.of( + new Schema.Field("field1", field1, null, null), + new Schema.Field("field2", field2, null, null) + )); + + StdType stdStructType = AvroWrapper.createStdType(structSchema); + assertTrue(stdStructType instanceof AvroStructType); + assertEquals(structSchema, stdStructType.underlyingType()); + assertEquals(field1, ((AvroStructType) stdStructType).fieldTypes().get(0).underlyingType()); + assertEquals(field2, ((AvroStructType) stdStructType).fieldTypes().get(1).underlyingType()); + + GenericRecord value = new GenericData.Record(structSchema); + value.put("field1", 1); + value.put("field2", 2.0); + StdData stdStructData = AvroWrapper.createStdData(value, structSchema); + assertTrue(stdStructData instanceof AvroStruct); + AvroStruct avroStruct = (AvroStruct) stdStructData; + assertEquals(2, avroStruct.fields().size()); + assertEquals(value, avroStruct.getUnderlyingData()); + assertEquals(1, ((PlatformData) avroStruct.getField("field1")).getUnderlyingData()); + assertEquals(2.0, ((PlatformData) avroStruct.getField("field2")).getUnderlyingData()); + } + + @Test + public void testValidUnionType() { + Schema nonNullType = createSchema("\"long\""); + Schema unionSchema = Schema.createUnion(Arrays.asList(nonNullType, Schema.create(Schema.Type.NULL))); + + StdType stdLongType = AvroWrapper.createStdType(unionSchema); + assertTrue(stdLongType instanceof AvroLongType); + assertEquals(nonNullType, stdLongType.underlyingType()); + + StdData stdLongData = AvroWrapper.createStdData(1L, unionSchema); + assertTrue(stdLongData instanceof AvroLong); + assertEquals(1L, ((AvroLong) stdLongData).get()); + + StdData stdNullData = AvroWrapper.createStdData(null, unionSchema); + assertNull(stdNullData); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testInvalidUnionType1() { + Schema nonNullType = createSchema("\"long\""); + Schema unionSchema = Schema.createUnion(Arrays.asList(nonNullType)); + AvroWrapper.createStdType(unionSchema); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testInvalidUnionType2() { + Schema nonNullType1 = createSchema("\"long\""); + Schema nonNullType2 = createSchema("\"int\""); + Schema unionSchema = Schema.createUnion(Arrays.asList(nonNullType1, nonNullType2)); + AvroWrapper.createStdData(1L, unionSchema); + } + + @Test + public void testStructWithSimpleUnionField() { + Schema field1 = createSchema("field1", "\"int\""); + Schema nonNullableField2 = createSchema("field2", "\"double\""); + Schema field2 = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), nonNullableField2)); + + Schema structSchema = Schema.createRecord(ImmutableList.of( + new Schema.Field("field1", field1, null, null), + new Schema.Field("field2", field2, null, null) + )); + + GenericRecord record1 = new GenericData.Record(structSchema); + record1.put("field1", 1); + record1.put("field2", 3.0); + AvroStruct avroStruct1 = (AvroStruct) AvroWrapper.createStdData(record1, structSchema); + assertEquals(2, avroStruct1.fields().size()); + assertEquals(3.0, ((PlatformData) avroStruct1.getField("field2")).getUnderlyingData()); + + GenericRecord record2 = new GenericData.Record(structSchema); + record2.put("field1", 1); + record2.put("field2", null); + AvroStruct avroStruct2 = (AvroStruct) AvroWrapper.createStdData(record2, structSchema); + assertEquals(2, avroStruct2.fields().size()); + assertNull(avroStruct2.getField("field2")); + assertNull(avroStruct2.fields().get(1)); + + GenericRecord record3 = new GenericData.Record(structSchema); + record3.put("field1", 1); + AvroStruct avroStruct3 = (AvroStruct) AvroWrapper.createStdData(record3, structSchema); + assertEquals(2, avroStruct3.fields().size()); + assertNull(avroStruct3.getField("field2")); + assertNull(avroStruct3.fields().get(1)); + } +} diff --git a/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/typesystem/TestAvroBoundVariables.java b/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/typesystem/TestAvroBoundVariables.java index 523d1d2a..9d2e3279 100644 --- a/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/typesystem/TestAvroBoundVariables.java +++ b/transportable-udfs-avro/src/test/java/com/linkedin/transport/avro/typesystem/TestAvroBoundVariables.java @@ -9,8 +9,10 @@ import com.linkedin.transport.typesystem.AbstractTestBoundVariables; import com.linkedin.transport.typesystem.AbstractTypeSystem; import org.apache.avro.Schema; +import org.testng.annotations.Test; +@Test public class TestAvroBoundVariables extends AbstractTestBoundVariables { @Override diff --git a/transportable-udfs-codegen/src/main/java/com/linkedin/transport/codegen/SparkWrapperGenerator.java b/transportable-udfs-codegen/src/main/java/com/linkedin/transport/codegen/SparkWrapperGenerator.java index bec08d8e..1bc19ded 100644 --- a/transportable-udfs-codegen/src/main/java/com/linkedin/transport/codegen/SparkWrapperGenerator.java +++ b/transportable-udfs-codegen/src/main/java/com/linkedin/transport/codegen/SparkWrapperGenerator.java @@ -11,7 +11,9 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Collection; +import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.text.StringSubstitutor; @@ -23,6 +25,7 @@ public class SparkWrapperGenerator implements WrapperGenerator { private static final String SPARK_WRAPPER_TEMPLATE_RESOURCE_PATH = "wrapper-templates/spark"; private static final String SUBSTITUTOR_KEY_WRAPPER_PACKAGE = "wrapperPackage"; private static final String SUBSTITUTOR_KEY_WRAPPER_CLASS = "wrapperClass"; + private static final String SUBSTITUTOR_KEY_WRAPPER_CLASS_PARAMERTERS = "wrapperClassParameters"; private static final String SUBSTITUTOR_KEY_UDF_TOP_LEVEL_CLASS = "udfTopLevelClass"; private static final String SUBSTITUTOR_KEY_UDF_IMPLEMENTATIONS = "udfImplementations"; @@ -30,12 +33,16 @@ public class SparkWrapperGenerator implements WrapperGenerator { public void generateWrappers(WrapperGeneratorContext context) { TransportUDFMetadata udfMetadata = context.getTransportUdfMetadata(); for (String topLevelClass : udfMetadata.getTopLevelClasses()) { - generateWrapper(topLevelClass, udfMetadata.getStdUDFImplementations(topLevelClass), + generateWrapper( + topLevelClass, + udfMetadata.getStdUDFImplementations(topLevelClass), + udfMetadata.getClassToNumberOfTypeParameters(), context.getSourcesOutputDir()); } } - private void generateWrapper(String topLevelClass, Collection implementationClasses, File outputDir) { + private void generateWrapper(String topLevelClass, Collection implementationClasses, + Map classToNumberOfTypeParameters, File outputDir) { final String wrapperTemplate; try (InputStream wrapperTemplateStream = Thread.currentThread() .getContextClassLoader() @@ -49,13 +56,15 @@ private void generateWrapper(String topLevelClass, Collection implementa ClassName wrapperClass = ClassName.get(topLevelClassName.packageName() + "." + SPARK_PACKAGE_SUFFIX, topLevelClassName.simpleName()); String udfImplementationInstantiations = implementationClasses.stream() - .map(clazz -> "new " + clazz + "()") + .map(clazz -> "new " + clazz + parameters(clazz, classToNumberOfTypeParameters) + "()") .collect(Collectors.joining(", ")); + String topLevelClassNameString = topLevelClassName.toString(); ImmutableMap substitutionMap = ImmutableMap.of( SUBSTITUTOR_KEY_WRAPPER_PACKAGE, wrapperClass.packageName(), SUBSTITUTOR_KEY_WRAPPER_CLASS, wrapperClass.simpleName(), - SUBSTITUTOR_KEY_UDF_TOP_LEVEL_CLASS, topLevelClassName.toString(), + SUBSTITUTOR_KEY_UDF_TOP_LEVEL_CLASS, topLevelClassNameString + + parameters(topLevelClassNameString, classToNumberOfTypeParameters), SUBSTITUTOR_KEY_UDF_IMPLEMENTATIONS, udfImplementationInstantiations ); @@ -69,4 +78,11 @@ private void generateWrapper(String topLevelClass, Collection implementa throw new RuntimeException("Error writing wrapper to file", e); } } + + private static String parameters(String clazz, Map classToNumberOfTypeParameters) { + int numberOfTypeParameters = classToNumberOfTypeParameters.get(clazz); + String[] objectTypes = new String[numberOfTypeParameters]; + Arrays.fill(objectTypes, "Object"); + return numberOfTypeParameters > 0 ? "[" + String.join(", ", objectTypes) + "]" : ""; + } } diff --git a/transportable-udfs-codegen/src/test/resources/inputs/sample-udf-metadata.json b/transportable-udfs-codegen/src/test/resources/inputs/sample-udf-metadata.json index 4d6fb8ae..6da584f2 100644 --- a/transportable-udfs-codegen/src/test/resources/inputs/sample-udf-metadata.json +++ b/transportable-udfs-codegen/src/test/resources/inputs/sample-udf-metadata.json @@ -1,17 +1,17 @@ { - "udfs": [ - { - "topLevelClass": "udfs.OverloadedUDF", - "stdUDFImplementations": [ - "udfs.OverloadedUDFInt", - "udfs.OverloadedUDFString" - ] - }, - { - "topLevelClass": "udfs.SimpleUDF", - "stdUDFImplementations": [ - "udfs.SimpleUDF" - ] - } - ] -} \ No newline at end of file + "udfs": { + "udfs.OverloadedUDF": [ + "udfs.OverloadedUDFInt", + "udfs.OverloadedUDFString" + ], + "udfs.SimpleUDF": [ + "udfs.SimpleUDF" + ] + }, + "classToNumberOfTypeParameters": { + "udfs.OverloadedUDFString": 0, + "udfs.OverloadedUDF": 0, + "udfs.OverloadedUDFInt": 0, + "udfs.SimpleUDF": 0 + } +} diff --git a/transportable-udfs-compile-utils/src/main/java/com/linkedin/transport/compile/TransportUDFMetadata.java b/transportable-udfs-compile-utils/src/main/java/com/linkedin/transport/compile/TransportUDFMetadata.java index 48db80f5..c10cc44b 100644 --- a/transportable-udfs-compile-utils/src/main/java/com/linkedin/transport/compile/TransportUDFMetadata.java +++ b/transportable-udfs-compile-utils/src/main/java/com/linkedin/transport/compile/TransportUDFMetadata.java @@ -9,14 +9,18 @@ import com.google.common.collect.Multimap; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.Reader; import java.io.Writer; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; import java.util.Set; @@ -26,6 +30,7 @@ public class TransportUDFMetadata { private static final Gson GSON; private Multimap _udfs; + private Map _classToNumberOfTypeParameters; static { GSON = new GsonBuilder().setPrettyPrinting().create(); @@ -33,14 +38,15 @@ public class TransportUDFMetadata { public TransportUDFMetadata() { _udfs = LinkedHashMultimap.create(); + _classToNumberOfTypeParameters = new HashMap<>(); } public void addUDF(String topLevelClass, String stdUDFImplementation) { _udfs.put(topLevelClass, stdUDFImplementation); } - public void addUDF(String topLevelClass, Collection stdUDFImplementations) { - _udfs.putAll(topLevelClass, stdUDFImplementations); + public void setClassNumberOfTypeParameters(String clazz, int numberOfTypeParameters) { + _classToNumberOfTypeParameters.put(clazz, numberOfTypeParameters); } public Set getTopLevelClasses() { @@ -51,8 +57,12 @@ public Collection getStdUDFImplementations(String topLevelClass) { return _udfs.get(topLevelClass); } + public Map getClassToNumberOfTypeParameters() { + return _classToNumberOfTypeParameters; + } + public void toJson(Writer writer) { - GSON.toJson(TransportUDFMetadataSerDe.fromUDFMetadata(this), writer); + GSON.toJson(TransportUDFMetadataSerDe.serialize(this), writer); } public static TransportUDFMetadata fromJsonFile(File jsonFile) { @@ -64,50 +74,49 @@ public static TransportUDFMetadata fromJsonFile(File jsonFile) { } public static TransportUDFMetadata fromJson(Reader reader) { - return TransportUDFMetadataSerDe.toUDFMetadata(GSON.fromJson(reader, TransportUDFMetadataJson.class)); + return TransportUDFMetadataSerDe.deserialize(new JsonParser().parse(reader)); } - /** - * Represents the JSON object structure of the Transport UDF metadata resource file - */ - private static class TransportUDFMetadataJson { - private List udfs; + private static class TransportUDFMetadataSerDe { - TransportUDFMetadataJson() { - this.udfs = new LinkedList<>(); + public static TransportUDFMetadata deserialize(JsonElement json) { + TransportUDFMetadata metadata = new TransportUDFMetadata(); + JsonObject root = json.getAsJsonObject(); + + // Deserialize udfs + JsonObject udfs = root.getAsJsonObject("udfs"); + udfs.keySet().forEach(topLevelClass -> { + JsonArray stdUdfImplementations = udfs.getAsJsonArray(topLevelClass); + for (int i = 0; i < stdUdfImplementations.size(); i++) { + metadata.addUDF(topLevelClass, stdUdfImplementations.get(i).getAsString()); + } + }); + + // Deserialize classToNumberOfTypeParameters + JsonObject classToNumberOfTypeParameters = root.getAsJsonObject("classToNumberOfTypeParameters"); + classToNumberOfTypeParameters.entrySet().forEach( + e -> metadata.setClassNumberOfTypeParameters(e.getKey(), e.getValue().getAsInt()) + ); + return metadata; } - static class UDFInfo { - private String topLevelClass; - private Collection stdUDFImplementations; - - UDFInfo(String topLevelClass, Collection stdUDFImplementations) { - this.topLevelClass = topLevelClass; - this.stdUDFImplementations = stdUDFImplementations; + public static JsonElement serialize(TransportUDFMetadata metadata) { + // Serialzie _udfs + JsonObject udfs = new JsonObject(); + for (Map.Entry> entry : metadata._udfs.asMap().entrySet()) { + JsonArray stdUdfImplementations = new JsonArray(); + entry.getValue().forEach(f -> stdUdfImplementations.add(f)); + udfs.add(entry.getKey(), stdUdfImplementations); } - } - } - /** - * Converts objects between {@link TransportUDFMetadata} and {@link TransportUDFMetadataJson} - */ - private static class TransportUDFMetadataSerDe { - - private static TransportUDFMetadataJson fromUDFMetadata(TransportUDFMetadata metadata) { - TransportUDFMetadataJson metadataJson = new TransportUDFMetadataJson(); - for (String topLevelClass : metadata.getTopLevelClasses()) { - metadataJson.udfs.add( - new TransportUDFMetadataJson.UDFInfo(topLevelClass, metadata.getStdUDFImplementations(topLevelClass))); - } - return metadataJson; - } + // Serialize _classToNumberOfTypeParameters + JsonObject classToNumberOfTypeParameters = new JsonObject(); + metadata._classToNumberOfTypeParameters.forEach((clazz, n) -> classToNumberOfTypeParameters.addProperty(clazz, n)); - private static TransportUDFMetadata toUDFMetadata(TransportUDFMetadataJson metadataJson) { - TransportUDFMetadata metadata = new TransportUDFMetadata(); - for (TransportUDFMetadataJson.UDFInfo udf : metadataJson.udfs) { - metadata.addUDF(udf.topLevelClass, udf.stdUDFImplementations); - } - return metadata; + JsonObject root = new JsonObject(); + root.add("udfs", udfs); + root.add("classToNumberOfTypeParameters", classToNumberOfTypeParameters); + return root; } } } diff --git a/transportable-udfs-examples/build.gradle b/transportable-udfs-examples/build.gradle index 240ba4e5..8714ba39 100644 --- a/transportable-udfs-examples/build.gradle +++ b/transportable-udfs-examples/build.gradle @@ -33,8 +33,8 @@ subprojects { url "https://conjars.org/repo" } } - project.ext.setProperty('presto-version', '319') - project.ext.setProperty('airlift-slice-version', '0.33') + project.ext.setProperty('presto-version', '333') + project.ext.setProperty('airlift-slice-version', '0.38') project.ext.setProperty('spark-group', 'org.apache.spark') project.ext.setProperty('spark-version', '2.3.0') } @@ -62,7 +62,9 @@ subprojects { } checkstyle { - configFile = file("${rootDir}/../gradle/checkstyle/linkedin-checkstyle.xml") + configFile = file("${rootDir}/../gradle/checkstyle/checkstyle.xml") + configProperties = ['config_loc' : "${rootDir}/../gradle/checkstyle/"] + toolVersion '8.23' } } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayElementAtFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayElementAtFunction.java index 2697f8db..e9ee2b22 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayElementAtFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayElementAtFunction.java @@ -6,15 +6,26 @@ package com.linkedin.transport.examples; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdInteger; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.api.udf.StdUDF2; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class ArrayElementAtFunction extends StdUDF2 implements TopLevelStdUDF { +/** + * Another way to define this class using generics can look like this + * + * public class ArrayElementAtFunction extends StdUDF2, Integer, K> implements TopLevelStdUDF { + * + * @Override + * public K eval(ArrayData a1, Integer idx) { + * return a1.get(idx); + * } + * + * } + * + */ +public class ArrayElementAtFunction extends StdUDF2 implements TopLevelStdUDF { @Override public String getFunctionName() { @@ -40,7 +51,7 @@ public String getOutputParameterSignature() { } @Override - public StdData eval(StdArray a1, StdInteger idx) { - return a1.get(idx.get()); + public Object eval(ArrayData a1, Integer idx) { + return a1.get(idx); } } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayFillFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayFillFunction.java index ae5a9ac1..a9cff404 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayFillFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/ArrayFillFunction.java @@ -7,16 +7,14 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdLong; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF2; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class ArrayFillFunction extends StdUDF2 implements TopLevelStdUDF { +public class ArrayFillFunction extends StdUDF2> implements TopLevelStdUDF { private StdType _arrayType; @@ -40,9 +38,9 @@ public void init(StdFactory stdFactory) { } @Override - public StdArray eval(StdData a, StdLong length) { - StdArray array = getStdFactory().createArray(_arrayType); - for (int i = 0; i < length.get(); i++) { + public ArrayData eval(K a, Long length) { + ArrayData array = getStdFactory().createArray(_arrayType); + for (int i = 0; i < length; i++) { array.add(a); } return array; diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryDuplicateFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryDuplicateFunction.java new file mode 100644 index 00000000..8252b816 --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryDuplicateFunction.java @@ -0,0 +1,46 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.api.udf.StdUDF1; +import com.linkedin.transport.api.udf.TopLevelStdUDF; +import java.nio.ByteBuffer; +import java.util.List; + + +public class BinaryDuplicateFunction extends StdUDF1 implements TopLevelStdUDF { + @Override + public ByteBuffer eval(ByteBuffer byteBuffer) { + ByteBuffer results = ByteBuffer.allocate(2 * byteBuffer.array().length); + for (int i = 0; i < 2; i++) { + for (byte b : byteBuffer.array()) { + results.put(b); + } + } + return results; + } + + @Override + public List getInputParameterSignatures() { + return ImmutableList.of("varbinary"); + } + + @Override + public String getOutputParameterSignature() { + return "varbinary"; + } + + @Override + public String getFunctionName() { + return "binary_duplicate"; + } + + @Override + public String getFunctionDescription() { + return "Duplicate a binary object"; + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryObjectSizeFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryObjectSizeFunction.java new file mode 100644 index 00000000..39b56cd4 --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/BinaryObjectSizeFunction.java @@ -0,0 +1,40 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.api.udf.StdUDF1; +import com.linkedin.transport.api.udf.TopLevelStdUDF; +import java.nio.ByteBuffer; +import java.util.List; + + +public class BinaryObjectSizeFunction extends StdUDF1 implements TopLevelStdUDF { + @Override + public Integer eval(ByteBuffer byteBuffer) { + return byteBuffer.array().length; + } + + @Override + public List getInputParameterSignatures() { + return ImmutableList.of("varbinary"); + } + + @Override + public String getOutputParameterSignature() { + return "integer"; + } + + @Override + public String getFunctionName() { + return "binary_size"; + } + + @Override + public String getFunctionDescription() { + return "Gets the size of a binary object"; + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/FileLookupFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/FileLookupFunction.java index 8112e443..e9ed378f 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/FileLookupFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/FileLookupFunction.java @@ -7,9 +7,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdBoolean; -import com.linkedin.transport.api.data.StdInteger; -import com.linkedin.transport.api.data.StdString; import com.linkedin.transport.api.udf.StdUDF2; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.io.BufferedReader; @@ -21,14 +18,14 @@ import org.apache.commons.io.IOUtils; -public class FileLookupFunction extends StdUDF2 implements TopLevelStdUDF { +public class FileLookupFunction extends StdUDF2 implements TopLevelStdUDF { private Set ids; @Override - public StdBoolean eval(StdString filename, StdInteger intToCheck) { + public Boolean eval(String filename, Integer intToCheck) { Preconditions.checkNotNull(intToCheck, "Integer to check should not be null"); - return getStdFactory().createBoolean(ids.contains(intToCheck.get())); + return ids.contains(intToCheck); } @Override @@ -57,8 +54,8 @@ public String getFunctionDescription() { } @Override - public String[] getRequiredFiles(StdString filename, StdInteger intToCheck) { - return new String[]{filename.get()}; + public String[] getRequiredFiles(String filename, Integer intToCheck) { + return new String[]{filename}; } public void processRequiredFiles(String[] localPaths) { diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapFromTwoArraysFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapFromTwoArraysFunction.java index 6fd99981..d6002a50 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapFromTwoArraysFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapFromTwoArraysFunction.java @@ -7,15 +7,16 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF2; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class MapFromTwoArraysFunction extends StdUDF2 implements TopLevelStdUDF { +public class MapFromTwoArraysFunction extends StdUDF2, ArrayData, MapData> + implements TopLevelStdUDF { private StdType _mapType; @@ -35,16 +36,16 @@ public String getOutputParameterSignature() { @Override public void init(StdFactory stdFactory) { super.init(stdFactory); - // Note: we create the _mapType once in init() and then reuse it to create StdMap objects + // Note: we create the _mapType once in init() and then reuse it to create MapData objects _mapType = getStdFactory().createStdType(getOutputParameterSignature()); } @Override - public StdMap eval(StdArray a1, StdArray a2) { + public MapData eval(ArrayData a1, ArrayData a2) { if (a1.size() != a2.size()) { return null; } - StdMap map = getStdFactory().createMap(_mapType); + MapData map = getStdFactory().createMap(_mapType); for (int i = 0; i < a1.size(); i++) { map.put(a1.get(i), a2.get(i)); } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapKeySetFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapKeySetFunction.java index a76c4403..af81e024 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapKeySetFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapKeySetFunction.java @@ -7,16 +7,15 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF1; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class MapKeySetFunction extends StdUDF1 implements TopLevelStdUDF { +public class MapKeySetFunction extends StdUDF1, ArrayData> implements TopLevelStdUDF { private StdType _mapType; @@ -39,9 +38,9 @@ public void init(StdFactory stdFactory) { } @Override - public StdArray eval(StdMap map) { - StdArray result = getStdFactory().createArray(_mapType); - for (StdData key : map.keySet()) { + public ArrayData eval(MapData map) { + ArrayData result = getStdFactory().createArray(_mapType); + for (K key : map.keySet()) { result.add(key); } return result; diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapValuesFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapValuesFunction.java index f22ff7f7..82b34ef1 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapValuesFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/MapValuesFunction.java @@ -7,16 +7,15 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF1; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class MapValuesFunction extends StdUDF1 implements TopLevelStdUDF { +public class MapValuesFunction extends StdUDF1, ArrayData> implements TopLevelStdUDF { private StdType _mapType; @@ -39,9 +38,9 @@ public void init(StdFactory stdFactory) { } @Override - public StdArray eval(StdMap map) { - StdArray result = getStdFactory().createArray(_mapType); - for (StdData value : map.values()) { + public ArrayData eval(MapData map) { + ArrayData result = getStdFactory().createArray(_mapType); + for (V value : map.values()) { result.add(value); } return result; diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddDoubleFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddDoubleFunction.java new file mode 100644 index 00000000..80b0fb2b --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddDoubleFunction.java @@ -0,0 +1,28 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.api.udf.StdUDF2; +import java.util.List; + + +public class NumericAddDoubleFunction extends StdUDF2 implements NumericAddFunction { + @Override + public Double eval(Double first, Double second) { + return first + second; + } + + @Override + public List getInputParameterSignatures() { + return ImmutableList.of("double", "double"); + } + + @Override + public String getOutputParameterSignature() { + return "double"; + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFloatFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFloatFunction.java new file mode 100644 index 00000000..a2a0ab47 --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFloatFunction.java @@ -0,0 +1,28 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.api.udf.StdUDF2; +import java.util.List; + + +public class NumericAddFloatFunction extends StdUDF2 implements NumericAddFunction { + @Override + public Float eval(Float first, Float second) { + return first + second; + } + + @Override + public List getInputParameterSignatures() { + return ImmutableList.of("real", "real"); + } + + @Override + public String getOutputParameterSignature() { + return "real"; + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFunction.java index 9c8e26d0..76e805e2 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddFunction.java @@ -17,6 +17,6 @@ default String getFunctionName() { @Override default String getFunctionDescription() { - return "Adds two integers or longs"; + return "Adds two integers, longs, reals, or doubles"; } } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddIntFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddIntFunction.java index cc5fb900..bcdb3696 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddIntFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddIntFunction.java @@ -6,16 +6,15 @@ package com.linkedin.transport.examples; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdInteger; import com.linkedin.transport.api.udf.StdUDF2; import java.util.List; -public class NumericAddIntFunction extends StdUDF2 +public class NumericAddIntFunction extends StdUDF2 implements NumericAddFunction { @Override - public StdInteger eval(StdInteger first, StdInteger second) { - return getStdFactory().createInteger(first.get() + second.get()); + public Integer eval(Integer first, Integer second) { + return first + second; } @Override diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddLongFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddLongFunction.java index a530e586..c24d2148 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddLongFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/NumericAddLongFunction.java @@ -6,15 +6,14 @@ package com.linkedin.transport.examples; import com.google.common.collect.ImmutableList; -import com.linkedin.transport.api.data.StdLong; import com.linkedin.transport.api.udf.StdUDF2; import java.util.List; -public class NumericAddLongFunction extends StdUDF2 implements NumericAddFunction { +public class NumericAddLongFunction extends StdUDF2 implements NumericAddFunction { @Override - public StdLong eval(StdLong first, StdLong second) { - return getStdFactory().createLong(first.get() + second.get()); + public Long eval(Long first, Long second) { + return first + second; } @Override diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByIndexFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByIndexFunction.java index 5a23283d..ffa78ba7 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByIndexFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByIndexFunction.java @@ -7,15 +7,14 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF2; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class StructCreateByIndexFunction extends StdUDF2 implements TopLevelStdUDF { +public class StructCreateByIndexFunction extends StdUDF2 implements TopLevelStdUDF { private StdType _field1Type; private StdType _field2Type; @@ -41,11 +40,11 @@ public void init(StdFactory stdFactory) { } @Override - public StdStruct eval(StdData field1Value, StdData field2Value) { - StdStruct struct = getStdFactory().createStruct(ImmutableList.of(_field1Type, _field2Type)); - struct.setField(0, field1Value); - struct.setField(1, field2Value); - return struct; + public RowData eval(Object field1Value, Object field2Value) { + RowData row = getStdFactory().createStruct(ImmutableList.of(_field1Type, _field2Type)); + row.setField(0, field1Value); + row.setField(1, field2Value); + return row; } @Override diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByNameFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByNameFunction.java index 36ca3472..b4f2a0c0 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByNameFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/main/java/com/linkedin/transport/examples/StructCreateByNameFunction.java @@ -7,16 +7,14 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdString; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.api.udf.StdUDF4; import com.linkedin.transport.api.udf.TopLevelStdUDF; import java.util.List; -public class StructCreateByNameFunction extends StdUDF4 implements TopLevelStdUDF { +public class StructCreateByNameFunction extends StdUDF4 implements TopLevelStdUDF { private StdType _field1Type; private StdType _field2Type; @@ -44,13 +42,13 @@ public void init(StdFactory stdFactory) { } @Override - public StdStruct eval(StdString field1Name, StdData field1Value, StdString field2Name, StdData field2Value) { - StdStruct struct = getStdFactory().createStruct( - ImmutableList.of(field1Name.get(), field2Name.get()), + public RowData eval(String field1Name, Object field1Value, String field2Name, Object field2Value) { + RowData struct = getStdFactory().createStruct( + ImmutableList.of(field1Name, field2Name), ImmutableList.of(_field1Type, _field2Type) ); - struct.setField(field1Name.get(), field1Value); - struct.setField(field2Name.get(), field2Value); + struct.setField(field1Name, field1Value); + struct.setField(field2Name, field2Value); return struct; } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java new file mode 100644 index 00000000..5e74e47c --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java @@ -0,0 +1,59 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.transport.api.udf.StdUDF; +import com.linkedin.transport.api.udf.TopLevelStdUDF; +import com.linkedin.transport.test.AbstractStdUDFTest; +import com.linkedin.transport.test.spi.StdTester; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import org.testng.annotations.Test; + + +public class TestBinaryDuplicateFunction extends AbstractStdUDFTest { + @Override + protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { + return ImmutableMap.of(BinaryDuplicateFunction.class, ImmutableList.of(BinaryDuplicateFunction.class)); + } + + @Test + public void testBinaryDuplicateASCII() { + StdTester tester = getTester(); + testBinaryDuplicateStringHelper(tester, "bar", "barbar"); + testBinaryDuplicateStringHelper(tester, "", ""); + testBinaryDuplicateStringHelper(tester, "foobar", "foobarfoobar"); + } + + @Test + public void testBinaryDuplicateUnicode() { + StdTester tester = getTester(); + testBinaryDuplicateStringHelper(tester, "こんにちは世界", "こんにちは世界こんにちは世界"); + testBinaryDuplicateStringHelper(tester, "\uD83D\uDE02", "\uD83D\uDE02\uD83D\uDE02"); + } + + private void testBinaryDuplicateStringHelper(StdTester tester, String input, String expectedOutput) { + ByteBuffer inputBuffer = ByteBuffer.wrap(input.getBytes()); + ByteBuffer expected = ByteBuffer.wrap(expectedOutput.getBytes()); + tester.check(functionCall("binary_duplicate", inputBuffer), expected, "varbinary"); + } + + @Test + public void testBinaryDuplicate() { + StdTester tester = getTester(); + testBinaryDuplicateHelper(tester, new byte[] {1, 2, 3}, new byte[] {1, 2, 3, 1, 2, 3}); + testBinaryDuplicateHelper(tester, new byte[] {-1, -2, -3}, new byte[] {-1, -2, -3, -1, -2, -3}); + } + + private void testBinaryDuplicateHelper(StdTester tester, byte[] input, byte[] expectedOutput) { + ByteBuffer inputBuffer = ByteBuffer.wrap(input); + ByteBuffer expected = ByteBuffer.wrap(expectedOutput); + tester.check(functionCall("binary_duplicate", inputBuffer), expected, "varbinary"); + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java new file mode 100644 index 00000000..b10bf0fc --- /dev/null +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java @@ -0,0 +1,36 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.examples; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.transport.api.udf.StdUDF; +import com.linkedin.transport.api.udf.TopLevelStdUDF; +import com.linkedin.transport.test.AbstractStdUDFTest; +import com.linkedin.transport.test.spi.StdTester; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import org.testng.annotations.Test; + + +public class TestBinaryObjectSizeFunction extends AbstractStdUDFTest { + @Override + protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { + return ImmutableMap.of(BinaryObjectSizeFunction.class, ImmutableList.of(BinaryObjectSizeFunction.class)); + } + + @Test + public void tesBinaryObjectSize() { + StdTester tester = getTester(); + ByteBuffer argTest1 = ByteBuffer.wrap("foo".getBytes()); + ByteBuffer argTest2 = ByteBuffer.wrap("".getBytes()); + ByteBuffer argTest3 = ByteBuffer.wrap("fooBar".getBytes()); + tester.check(functionCall("binary_size", argTest1), 3, "integer"); + tester.check(functionCall("binary_size", argTest2), 0, "integer"); + tester.check(functionCall("binary_size", argTest3), 6, "integer"); + } +} diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNumericAddFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNumericAddFunction.java index 8114d7e4..12f9791e 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNumericAddFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNumericAddFunction.java @@ -21,7 +21,11 @@ public class TestNumericAddFunction extends AbstractStdUDFTest { @Override protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { return ImmutableMap.of(NumericAddFunction.class, - ImmutableList.of(NumericAddIntFunction.class, NumericAddLongFunction.class)); + ImmutableList.of( + NumericAddIntFunction.class, + NumericAddLongFunction.class, + NumericAddFloatFunction.class, + NumericAddDoubleFunction.class)); } @Test @@ -29,5 +33,15 @@ public void testNumericAdd() { StdTester tester = getTester(); tester.check(functionCall("numeric_add", 1, 2), 3, "integer"); tester.check(functionCall("numeric_add", 1L, 2L), 3L, "bigint"); + tester.check(functionCall("numeric_add", 3.0, 4.0), 7.0, "double"); + + Object expectedResult; + if (tester.getClass().getCanonicalName().contains("HiveTester")) { + // Note that org.apache.hive.service.cli.Column.addValue() converts any elements in RowSet from float to double + expectedResult = 5.0; + } else { + expectedResult = 5.0f; + } + tester.check(functionCall("numeric_add", 2.0f, 3.0f), expectedResult, "real"); } } diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveFactory.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveFactory.java index 18b81f03..c058e56b 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveFactory.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveFactory.java @@ -5,23 +5,14 @@ */ package com.linkedin.transport.hive; -import com.google.common.base.Preconditions; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdBoolean; -import com.linkedin.transport.api.data.StdInteger; -import com.linkedin.transport.api.data.StdLong; -import com.linkedin.transport.api.data.StdMap; -import com.linkedin.transport.api.data.StdString; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.hive.data.HiveArray; -import com.linkedin.transport.hive.data.HiveBoolean; -import com.linkedin.transport.hive.data.HiveInteger; -import com.linkedin.transport.hive.data.HiveLong; -import com.linkedin.transport.hive.data.HiveMap; -import com.linkedin.transport.hive.data.HiveString; -import com.linkedin.transport.hive.data.HiveStruct; +import com.linkedin.transport.hive.data.HiveArrayData; +import com.linkedin.transport.hive.data.HiveMapData; +import com.linkedin.transport.hive.data.HiveRowData; import com.linkedin.transport.hive.types.objectinspector.CacheableObjectInspectorConverters; import com.linkedin.transport.hive.typesystem.HiveTypeFactory; import com.linkedin.transport.typesystem.AbstractBoundVariables; @@ -38,7 +29,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; public class HiveFactory implements StdFactory { @@ -54,44 +44,23 @@ public HiveFactory(AbstractBoundVariables boundVariables) { } @Override - public StdInteger createInteger(int value) { - return new HiveInteger(value, PrimitiveObjectInspectorFactory.javaIntObjectInspector, this); - } - - @Override - public StdLong createLong(long value) { - return new HiveLong(value, PrimitiveObjectInspectorFactory.javaLongObjectInspector, this); - } - - @Override - public StdBoolean createBoolean(boolean value) { - return new HiveBoolean(value, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, this); - } - - @Override - public StdString createString(String value) { - Preconditions.checkNotNull(value, "Cannot create a null StdString"); - return new HiveString(value, PrimitiveObjectInspectorFactory.javaStringObjectInspector, this); - } - - @Override - public StdArray createArray(StdType stdType, int expectedSize) { + public ArrayData createArray(StdType stdType, int expectedSize) { ListObjectInspector listObjectInspector = (ListObjectInspector) stdType.underlyingType(); - return new HiveArray( + return new HiveArrayData( new ArrayList(expectedSize), ObjectInspectorFactory.getStandardListObjectInspector(listObjectInspector.getListElementObjectInspector()), this); } @Override - public StdArray createArray(StdType stdType) { + public ArrayData createArray(StdType stdType) { return createArray(stdType, 0); } @Override - public StdMap createMap(StdType stdType) { + public MapData createMap(StdType stdType) { MapObjectInspector mapObjectInspector = (MapObjectInspector) stdType.underlyingType(); - return new HiveMap( + return new HiveMapData( new HashMap(), ObjectInspectorFactory.getStandardMapObjectInspector( mapObjectInspector.getMapKeyObjectInspector(), @@ -100,8 +69,8 @@ public StdMap createMap(StdType stdType) { } @Override - public StdStruct createStruct(List fieldNames, List fieldTypes) { - return new HiveStruct( + public RowData createStruct(List fieldNames, List fieldTypes) { + return new HiveRowData( new ArrayList(Arrays.asList(new Object[fieldTypes.size()])), ObjectInspectorFactory.getStandardStructObjectInspector( fieldNames, @@ -111,16 +80,16 @@ public StdStruct createStruct(List fieldNames, List fieldTypes) } @Override - public StdStruct createStruct(List fieldTypes) { + public RowData createStruct(List fieldTypes) { List fieldNames = IntStream.range(0, fieldTypes.size()).mapToObj(i -> "field" + i).collect(Collectors.toList()); return createStruct(fieldNames, fieldTypes); } @Override - public StdStruct createStruct(StdType stdType) { + public RowData createStruct(StdType stdType) { StructObjectInspector structObjectInspector = (StructObjectInspector) stdType.underlyingType(); - return new HiveStruct( + return new HiveRowData( new ArrayList(Arrays.asList(new Object[structObjectInspector.getAllStructFieldRefs().size()])), ObjectInspectorFactory.getStandardStructObjectInspector( structObjectInspector.getAllStructFieldRefs() diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveWrapper.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveWrapper.java index b2980836..2b06d7a4 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveWrapper.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/HiveWrapper.java @@ -6,30 +6,42 @@ package com.linkedin.transport.hive; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.hive.data.HiveArray; -import com.linkedin.transport.hive.data.HiveBoolean; -import com.linkedin.transport.hive.data.HiveInteger; -import com.linkedin.transport.hive.data.HiveLong; -import com.linkedin.transport.hive.data.HiveMap; -import com.linkedin.transport.hive.data.HiveString; -import com.linkedin.transport.hive.data.HiveStruct; +import com.linkedin.transport.hive.data.HiveArrayData; +import com.linkedin.transport.hive.data.HiveData; +import com.linkedin.transport.hive.data.HiveMapData; +import com.linkedin.transport.hive.data.HiveRowData; import com.linkedin.transport.hive.types.HiveArrayType; import com.linkedin.transport.hive.types.HiveBooleanType; +import com.linkedin.transport.hive.types.HiveBinaryType; +import com.linkedin.transport.hive.types.HiveDoubleType; +import com.linkedin.transport.hive.types.HiveFloatType; import com.linkedin.transport.hive.types.HiveIntegerType; import com.linkedin.transport.hive.types.HiveLongType; import com.linkedin.transport.hive.types.HiveMapType; import com.linkedin.transport.hive.types.HiveStringType; -import com.linkedin.transport.hive.types.HiveStructType; +import com.linkedin.transport.hive.types.HiveRowType; import com.linkedin.transport.hive.types.HiveUnknownType; +import java.nio.ByteBuffer; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; @@ -39,22 +51,22 @@ public final class HiveWrapper { private HiveWrapper() { } - public static StdData createStdData(Object hiveData, ObjectInspector hiveObjectInspector, StdFactory stdFactory) { - if (hiveObjectInspector instanceof IntObjectInspector) { - return new HiveInteger(hiveData, (IntObjectInspector) hiveObjectInspector, stdFactory); - } else if (hiveObjectInspector instanceof LongObjectInspector) { - return new HiveLong(hiveData, (LongObjectInspector) hiveObjectInspector, stdFactory); - } else if (hiveObjectInspector instanceof BooleanObjectInspector) { - return new HiveBoolean(hiveData, (BooleanObjectInspector) hiveObjectInspector, stdFactory); - } else if (hiveObjectInspector instanceof StringObjectInspector) { - return new HiveString(hiveData, (StringObjectInspector) hiveObjectInspector, stdFactory); + public static Object createStdData(Object hiveData, ObjectInspector hiveObjectInspector, StdFactory stdFactory) { + if (hiveObjectInspector instanceof IntObjectInspector || hiveObjectInspector instanceof LongObjectInspector + || hiveObjectInspector instanceof FloatObjectInspector || hiveObjectInspector instanceof DoubleObjectInspector + || hiveObjectInspector instanceof BooleanObjectInspector + || hiveObjectInspector instanceof StringObjectInspector) { + return ((PrimitiveObjectInspector) hiveObjectInspector).getPrimitiveJavaObject(hiveData); + } else if (hiveObjectInspector instanceof BinaryObjectInspector) { + BinaryObjectInspector binaryObjectInspector = (BinaryObjectInspector) hiveObjectInspector; + return hiveData == null ? null : ByteBuffer.wrap(binaryObjectInspector.getPrimitiveJavaObject(hiveData)); } else if (hiveObjectInspector instanceof ListObjectInspector) { ListObjectInspector listObjectInspector = (ListObjectInspector) hiveObjectInspector; - return new HiveArray(hiveData, listObjectInspector, stdFactory); + return new HiveArrayData(hiveData, listObjectInspector, stdFactory); } else if (hiveObjectInspector instanceof MapObjectInspector) { - return new HiveMap(hiveData, hiveObjectInspector, stdFactory); + return new HiveMapData(hiveData, hiveObjectInspector, stdFactory); } else if (hiveObjectInspector instanceof StructObjectInspector) { - return new HiveStruct(((StructObjectInspector) hiveObjectInspector).getStructFieldsDataAsList(hiveData).toArray(), + return new HiveRowData(((StructObjectInspector) hiveObjectInspector).getStructFieldsDataAsList(hiveData).toArray(), hiveObjectInspector, stdFactory); } else if (hiveObjectInspector instanceof VoidObjectInspector) { return null; @@ -72,16 +84,68 @@ public static StdType createStdType(ObjectInspector hiveObjectInspector) { return new HiveBooleanType((BooleanObjectInspector) hiveObjectInspector); } else if (hiveObjectInspector instanceof StringObjectInspector) { return new HiveStringType((StringObjectInspector) hiveObjectInspector); + } else if (hiveObjectInspector instanceof FloatObjectInspector) { + return new HiveFloatType((FloatObjectInspector) hiveObjectInspector); + } else if (hiveObjectInspector instanceof DoubleObjectInspector) { + return new HiveDoubleType((DoubleObjectInspector) hiveObjectInspector); + } else if (hiveObjectInspector instanceof BinaryObjectInspector) { + return new HiveBinaryType((BinaryObjectInspector) hiveObjectInspector); } else if (hiveObjectInspector instanceof ListObjectInspector) { return new HiveArrayType((ListObjectInspector) hiveObjectInspector); } else if (hiveObjectInspector instanceof MapObjectInspector) { return new HiveMapType((MapObjectInspector) hiveObjectInspector); } else if (hiveObjectInspector instanceof StructObjectInspector) { - return new HiveStructType((StructObjectInspector) hiveObjectInspector); + return new HiveRowType((StructObjectInspector) hiveObjectInspector); } else if (hiveObjectInspector instanceof VoidObjectInspector) { return new HiveUnknownType((VoidObjectInspector) hiveObjectInspector); } assert false : "Unrecognized Hive ObjectInspector: " + hiveObjectInspector.getClass(); return null; } + + public static Object getPlatformDataForObjectInspector(Object transportData, ObjectInspector oi) { + if (transportData == null) { + return null; + } else if (oi instanceof IntObjectInspector) { + return ((SettableIntObjectInspector) oi).create((Integer) transportData); + } else if (oi instanceof LongObjectInspector) { + return ((SettableLongObjectInspector) oi).create((Long) transportData); + } else if (oi instanceof FloatObjectInspector) { + return ((SettableFloatObjectInspector) oi).create((Float) transportData); + } else if (oi instanceof DoubleObjectInspector) { + return ((SettableDoubleObjectInspector) oi).create((Double) transportData); + } else if (oi instanceof BooleanObjectInspector) { + return ((SettableBooleanObjectInspector) oi).create((Boolean) transportData); + } else if (oi instanceof StringObjectInspector) { + return ((SettableStringObjectInspector) oi).create((String) transportData); + } else if (oi instanceof BinaryObjectInspector) { + return ((SettableBinaryObjectInspector) oi).create(((ByteBuffer) transportData).array()); + } else { + return ((HiveData) transportData).getUnderlyingDataForObjectInspector(oi); + } + } + + public static Object getStandardObject(Object transportData) { + if (transportData == null) { + return null; + } else if (transportData instanceof Integer) { + return PrimitiveObjectInspectorFactory.writableIntObjectInspector.create((Integer) transportData); + } else if (transportData instanceof Long) { + return PrimitiveObjectInspectorFactory.writableLongObjectInspector.create((Long) transportData); + } else if (transportData instanceof Float) { + return PrimitiveObjectInspectorFactory.writableFloatObjectInspector.create((Float) transportData); + } else if (transportData instanceof Double) { + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.create((Double) transportData); + } else if (transportData instanceof Boolean) { + return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector.create((Boolean) transportData); + } else if (transportData instanceof String) { + return PrimitiveObjectInspectorFactory.writableStringObjectInspector.create((String) transportData); + } else if (transportData instanceof ByteBuffer) { + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create(((ByteBuffer) transportData).array()); + } else { + return ((HiveData) transportData).getUnderlyingDataForObjectInspector( + ((HiveData) transportData).getUnderlyingObjectInspector() + ); + } + } } diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/StdUdfWrapper.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/StdUdfWrapper.java index 3b3da9ab..5f14689e 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/StdUdfWrapper.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/StdUdfWrapper.java @@ -7,7 +7,6 @@ import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.StdUDF1; @@ -23,6 +22,7 @@ import com.linkedin.transport.utils.FileSystemUtils; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.IntStream; @@ -35,6 +35,9 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; /** @@ -49,7 +52,8 @@ public abstract class StdUdfWrapper extends GenericUDF { protected StdFactory _stdFactory; private boolean[] _nullableArguments; private String[] _distributedCacheFiles; - private StdData[] _args; + private Object[] _args; + private ObjectInspector _outputObjectInspector; /** * Given input object inspectors, this method matches them to the expected type signatures, and finds bindings to the @@ -70,7 +74,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) { _stdUdf.init(_stdFactory); _requiredFilesProcessed = false; createStdData(); - return hiveTypeInference.getOutputDataType(); + _outputObjectInspector= hiveTypeInference.getOutputDataType(); + return _outputObjectInspector; } @Override @@ -108,14 +113,23 @@ protected boolean containsNullValuedNonNullableConstants() { return false; } - protected StdData wrap(DeferredObject hiveDeferredObject, StdData stdData) { + protected Object wrap(DeferredObject hiveDeferredObject, ObjectInspector inputObjectInspector, Object stdData) { try { Object hiveObject = hiveDeferredObject.get(); - if (hiveObject != null) { - ((PlatformData) stdData).setUnderlyingData(hiveObject); - return stdData; + if (inputObjectInspector instanceof BinaryObjectInspector) { + return hiveObject == null ? null : ByteBuffer.wrap( + ((BinaryObjectInspector) inputObjectInspector).getPrimitiveJavaObject(hiveObject) + ); + } + if (inputObjectInspector instanceof PrimitiveObjectInspector) { + return ((PrimitiveObjectInspector) inputObjectInspector).getPrimitiveJavaObject(hiveObject); } else { - return null; + if (hiveObject != null) { + ((PlatformData) stdData).setUnderlyingData(hiveObject); + return stdData; + } else { + return null; + } } } catch (HiveException e) { throw new RuntimeException("Cannot extract Hive Object from Deferred Object"); @@ -127,21 +141,35 @@ protected StdData wrap(DeferredObject hiveDeferredObject, StdData stdData) { protected abstract Class getTopLevelUdfClass(); protected void createStdData() { - _args = new StdData[_inputObjectInspectors.length]; + _args = new Object[_inputObjectInspectors.length]; for (int i = 0; i < _inputObjectInspectors.length; i++) { _args[i] = HiveWrapper.createStdData(null, _inputObjectInspectors[i], _stdFactory); } } - private StdData[] wrapArguments(DeferredObject[] deferredObjects) { - return IntStream.range(0, _args.length).mapToObj(i -> wrap(deferredObjects[i], _args[i])).toArray(StdData[]::new); + private Object getPlatformData(Object transportData) { + if (transportData == null) { + return null; + } else if (transportData instanceof Integer || transportData instanceof Long || transportData instanceof Boolean + || transportData instanceof String || transportData instanceof Float || transportData instanceof Double || + transportData instanceof ByteBuffer) { + return HiveWrapper.getPlatformDataForObjectInspector(transportData, _outputObjectInspector); + } else { + return ((PlatformData) transportData).getUnderlyingData(); + } + } + + private Object[] wrapArguments(DeferredObject[] deferredObjects) { + return IntStream.range(0, _args.length).mapToObj( + i -> wrap(deferredObjects[i], _inputObjectInspectors[i], _args[i]) + ).toArray(Object[]::new); } - private StdData[] wrapConstants() { + private Object[] wrapConstants() { return Arrays.stream(_inputObjectInspectors) .map(oi -> (oi instanceof ConstantObjectInspector) ? HiveWrapper.createStdData( ((ConstantObjectInspector) oi).getWritableConstantValue(), oi, _stdFactory) : null) - .toArray(StdData[]::new); + .toArray(Object[]::new); } @Override @@ -152,8 +180,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { if (!_requiredFilesProcessed) { processRequiredFiles(); } - StdData[] args = wrapArguments(arguments); - StdData result; + Object[] args = wrapArguments(arguments); + Object result; switch (args.length) { case 0: result = ((StdUDF0) _stdUdf).eval(); @@ -185,7 +213,7 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { default: throw new UnsupportedOperationException("eval not yet supported for StdUDF" + args.length); } - return result == null ? null : ((PlatformData) result).getUnderlyingData(); + return getPlatformData(result); } @Override @@ -193,7 +221,7 @@ public String[] getRequiredFiles() { if (containsNullValuedNonNullableConstants()) { return new String[]{}; } - StdData[] args = wrapConstants(); + Object[] args = wrapConstants(); String[] requiredFiles; switch (args.length) { case 0: @@ -231,7 +259,7 @@ public String[] getRequiredFiles() { } _distributedCacheFiles = Arrays.stream(requiredFiles).map(requiredFile -> { try { - return FileSystemUtils.resolveLatest(requiredFile, FileSystemUtils.getHDFSFileSystem()); + return FileSystemUtils.resolveLatest(requiredFile); } catch (IOException e) { throw new RuntimeException("Failed to resolve path: [" + requiredFile + "].", e); } diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArray.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArrayData.java similarity index 72% rename from transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArray.java rename to transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArrayData.java index 57cb0e8c..d0bf8ec4 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArray.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveArrayData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.hive.data; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.hive.HiveWrapper; import java.util.Iterator; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -15,12 +14,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; -public class HiveArray extends HiveData implements StdArray { +public class HiveArrayData extends HiveData implements ArrayData { final ListObjectInspector _listObjectInspector; final ObjectInspector _elementObjectInspector; - public HiveArray(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { + public HiveArrayData(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { super(stdFactory); _object = object; _listObjectInspector = (ListObjectInspector) objectInspector; @@ -33,19 +32,21 @@ public int size() { } @Override - public StdData get(int idx) { - return HiveWrapper.createStdData(_listObjectInspector.getListElement(_object, idx), _elementObjectInspector, + public E get(int idx) { + return (E) HiveWrapper.createStdData( + _listObjectInspector.getListElement(_object, idx), + _elementObjectInspector, _stdFactory); } @Override - public void add(StdData e) { + public void add(E e) { if (_listObjectInspector instanceof SettableListObjectInspector) { SettableListObjectInspector settableListObjectInspector = (SettableListObjectInspector) _listObjectInspector; int originalSize = size(); settableListObjectInspector.resize(_object, originalSize + 1); settableListObjectInspector.set(_object, originalSize, - ((HiveData) e).getUnderlyingDataForObjectInspector(_elementObjectInspector)); + HiveWrapper.getPlatformDataForObjectInspector(e, _elementObjectInspector)); _isObjectModified = true; } else { throw new RuntimeException("Attempt to modify an immutable Hive object of type: " @@ -59,8 +60,8 @@ public ObjectInspector getUnderlyingObjectInspector() { } @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { int size = size(); int currentIndex = 0; @@ -70,8 +71,8 @@ public boolean hasNext() { } @Override - public StdData next() { - StdData element = HiveWrapper.createStdData(_listObjectInspector.getListElement(_object, currentIndex), + public E next() { + E element = (E) HiveWrapper.createStdData(_listObjectInspector.getListElement(_object, currentIndex), _elementObjectInspector, _stdFactory); currentIndex++; return element; diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveBoolean.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveBoolean.java deleted file mode 100644 index b4537170..00000000 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveBoolean.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.hive.data; - -import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdBoolean; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; - - -public class HiveBoolean extends HiveData implements StdBoolean { - - final BooleanObjectInspector _booleanObjectInspector; - - public HiveBoolean(Object object, BooleanObjectInspector booleanObjectInspector, StdFactory stdFactory) { - super(stdFactory); - _object = object; - _booleanObjectInspector = booleanObjectInspector; - } - - @Override - public boolean get() { - return _booleanObjectInspector.get(_object); - } - - @Override - public ObjectInspector getUnderlyingObjectInspector() { - return _booleanObjectInspector; - } -} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveData.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveData.java index 51beb456..94337266 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveData.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveData.java @@ -59,10 +59,6 @@ public ObjectInspector getStandardObjectInspector() { getUnderlyingObjectInspector(), ObjectInspectorUtils.ObjectInspectorCopyOption.WRITABLE); } - public Object getStandardObject() { - return getUnderlyingDataForObjectInspector(getStandardObjectInspector()); - } - private Object getObjectFromCache(ObjectInspector oi) { if (_isObjectModified) { _cachedObjectsForObjectInspectors.clear(); diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveInteger.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveInteger.java deleted file mode 100644 index a1d2e38f..00000000 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveInteger.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.hive.data; - -import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdInteger; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; - - -public class HiveInteger extends HiveData implements StdInteger { - - final IntObjectInspector _intObjectInspector; - - public HiveInteger(Object object, IntObjectInspector intObjectInspector, StdFactory stdFactory) { - super(stdFactory); - _object = object; - _intObjectInspector = intObjectInspector; - } - - @Override - public int get() { - return _intObjectInspector.get(_object); - } - - @Override - public ObjectInspector getUnderlyingObjectInspector() { - return _intObjectInspector; - } -} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveLong.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveLong.java deleted file mode 100644 index 0b662b59..00000000 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveLong.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.hive.data; - -import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdLong; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; - - -public class HiveLong extends HiveData implements StdLong { - - final LongObjectInspector _longObjectInspector; - - public HiveLong(Object object, LongObjectInspector longObjectInspector, StdFactory stdFactory) { - super(stdFactory); - _object = object; - _longObjectInspector = longObjectInspector; - } - - @Override - public long get() { - return _longObjectInspector.get(_object); - } - - @Override - public ObjectInspector getUnderlyingObjectInspector() { - return _longObjectInspector; - } -} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMap.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMapData.java similarity index 66% rename from transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMap.java rename to transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMapData.java index 70f5132b..54da6042 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMap.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveMapData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.hive.data; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.hive.HiveWrapper; import java.util.AbstractCollection; import java.util.AbstractSet; @@ -20,13 +19,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; -public class HiveMap extends HiveData implements StdMap { +public class HiveMapData extends HiveData implements MapData { final MapObjectInspector _mapObjectInspector; final ObjectInspector _keyObjectInspector; final ObjectInspector _valueObjectInspector; - public HiveMap(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { + public HiveMapData(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { super(stdFactory); _object = object; _mapObjectInspector = (MapObjectInspector) objectInspector; @@ -40,30 +39,30 @@ public int size() { } @Override - public StdData get(StdData key) { + public V get(K key) { MapObjectInspector mapOI = _mapObjectInspector; Object mapObj = _object; Object keyObj; try { - keyObj = ((HiveData) key).getUnderlyingDataForObjectInspector(_keyObjectInspector); + keyObj = HiveWrapper.getPlatformDataForObjectInspector(key, _keyObjectInspector); } catch (RuntimeException e) { // Cannot convert key argument to Map's KeyOI. So convert both the map and the key arg to // objects having standard OIs mapOI = (MapObjectInspector) getStandardObjectInspector(); - mapObj = getStandardObject(); - keyObj = ((HiveData) key).getStandardObject(); + mapObj = HiveWrapper.getStandardObject(this); + keyObj = HiveWrapper.getStandardObject(key); } - return HiveWrapper.createStdData( + return (V) HiveWrapper.createStdData( mapOI.getMapValueElement(mapObj, keyObj), mapOI.getMapValueObjectInspector(), _stdFactory); } @Override - public void put(StdData key, StdData value) { + public void put(K key, V value) { if (_mapObjectInspector instanceof SettableMapObjectInspector) { - Object keyObj = ((HiveData) key).getUnderlyingDataForObjectInspector(_keyObjectInspector); - Object valueObj = ((HiveData) value).getUnderlyingDataForObjectInspector(_valueObjectInspector); + Object keyObj = HiveWrapper.getPlatformDataForObjectInspector(key, _keyObjectInspector); + Object valueObj = HiveWrapper.getPlatformDataForObjectInspector(value, _valueObjectInspector); ((SettableMapObjectInspector) _mapObjectInspector).put( _object, @@ -79,11 +78,11 @@ public void put(StdData key, StdData value) { //TODO: Cache the result of .getMap(_object) below for subsequent calls. @Override - public Set keySet() { - return new AbstractSet() { + public Set keySet() { + return new AbstractSet() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator mapKeyIterator = _mapObjectInspector.getMap(_object).keySet().iterator(); @Override @@ -92,26 +91,26 @@ public boolean hasNext() { } @Override - public StdData next() { - return HiveWrapper.createStdData(mapKeyIterator.next(), _keyObjectInspector, _stdFactory); + public K next() { + return (K) HiveWrapper.createStdData(mapKeyIterator.next(), _keyObjectInspector, _stdFactory); } }; } @Override public int size() { - return HiveMap.this.size(); + return HiveMapData.this.size(); } }; } //TODO: Cache the result of .getMap(_object) below for subsequent calls. @Override - public Collection values() { - return new AbstractCollection() { + public Collection values() { + return new AbstractCollection() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator mapValueIterator = _mapObjectInspector.getMap(_object).values().iterator(); @Override @@ -120,30 +119,30 @@ public boolean hasNext() { } @Override - public StdData next() { - return HiveWrapper.createStdData(mapValueIterator.next(), _valueObjectInspector, _stdFactory); + public V next() { + return (V) HiveWrapper.createStdData(mapValueIterator.next(), _valueObjectInspector, _stdFactory); } }; } @Override public int size() { - return HiveMap.this.size(); + return HiveMapData.this.size(); } }; } @Override - public boolean containsKey(StdData key) { + public boolean containsKey(K key) { Object mapObj = _object; Object keyObj; try { - keyObj = ((HiveData) key).getUnderlyingDataForObjectInspector(_keyObjectInspector); + keyObj = HiveWrapper.getPlatformDataForObjectInspector(key, _keyObjectInspector); } catch (RuntimeException e) { // Cannot convert key argument to Map's KeyOI. So convertboth the map and the key arg to // objects having standard OIs - mapObj = getStandardObject(); - keyObj = ((HiveData) key).getStandardObject(); + mapObj = HiveWrapper.getStandardObject(this); + keyObj = HiveWrapper.getStandardObject(key); } return ((Map) mapObj).containsKey(keyObj); diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveStruct.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveRowData.java similarity index 79% rename from transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveStruct.java rename to transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveRowData.java index 80872eff..5704374e 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveStruct.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveRowData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.hive.data; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.hive.HiveWrapper; import java.util.List; import java.util.stream.Collectors; @@ -18,18 +17,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -public class HiveStruct extends HiveData implements StdStruct { +public class HiveRowData extends HiveData implements RowData { StructObjectInspector _structObjectInspector; - public HiveStruct(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { + public HiveRowData(Object object, ObjectInspector objectInspector, StdFactory stdFactory) { super(stdFactory); _object = object; _structObjectInspector = (StructObjectInspector) objectInspector; } @Override - public StdData getField(int index) { + public Object getField(int index) { StructField structField = _structObjectInspector.getAllStructFieldRefs().get(index); return HiveWrapper.createStdData( _structObjectInspector.getStructFieldData(_object, structField), @@ -38,7 +37,7 @@ public StdData getField(int index) { } @Override - public StdData getField(String name) { + public Object getField(String name) { StructField structField = _structObjectInspector.getStructFieldRef(name); return HiveWrapper.createStdData( _structObjectInspector.getStructFieldData(_object, structField), @@ -47,11 +46,11 @@ public StdData getField(String name) { } @Override - public void setField(int index, StdData value) { + public void setField(int index, Object value) { if (_structObjectInspector instanceof SettableStructObjectInspector) { StructField field = _structObjectInspector.getAllStructFieldRefs().get(index); ((SettableStructObjectInspector) _structObjectInspector).setStructFieldData(_object, - field, ((HiveData) value).getUnderlyingDataForObjectInspector(field.getFieldObjectInspector()) + field, HiveWrapper.getPlatformDataForObjectInspector(value, field.getFieldObjectInspector()) ); _isObjectModified = true; } else { @@ -61,11 +60,11 @@ public void setField(int index, StdData value) { } @Override - public void setField(String name, StdData value) { + public void setField(String name, Object value) { if (_structObjectInspector instanceof SettableStructObjectInspector) { StructField field = _structObjectInspector.getStructFieldRef(name); ((SettableStructObjectInspector) _structObjectInspector).setStructFieldData(_object, - field, ((HiveData) value).getUnderlyingDataForObjectInspector(field.getFieldObjectInspector())); + field, HiveWrapper.getPlatformDataForObjectInspector(value, field.getFieldObjectInspector())); _isObjectModified = true; } else { throw new RuntimeException("Attempt to modify an immutable Hive object of type: " @@ -74,7 +73,7 @@ public void setField(String name, StdData value) { } @Override - public List fields() { + public List fields() { return IntStream.range(0, _structObjectInspector.getAllStructFieldRefs().size()).mapToObj(i -> getField(i)) .collect(Collectors.toList()); } diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveString.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveString.java deleted file mode 100644 index 83310309..00000000 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/data/HiveString.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.hive.data; - -import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdString; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; - - -public class HiveString extends HiveData implements StdString { - - final StringObjectInspector _stringObjectInspector; - - public HiveString(Object object, StringObjectInspector stringObjectInspector, StdFactory stdFactory) { - super(stdFactory); - _object = object; - _stringObjectInspector = stringObjectInspector; - } - - @Override - public String get() { - return _stringObjectInspector.getPrimitiveJavaObject(_object); - } - - @Override - public ObjectInspector getUnderlyingObjectInspector() { - return _stringObjectInspector; - } -} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveBinaryType.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveBinaryType.java new file mode 100644 index 00000000..bc21a5d7 --- /dev/null +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveBinaryType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.hive.types; + +import com.linkedin.transport.api.types.StdBinaryType; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; + + +public class HiveBinaryType implements StdBinaryType { + + private final BinaryObjectInspector _binaryObjectInspector; + + public HiveBinaryType(BinaryObjectInspector binaryObjectInspector) { + _binaryObjectInspector = binaryObjectInspector; + } + + @Override + public Object underlyingType() { + return _binaryObjectInspector; + } +} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveDoubleType.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveDoubleType.java new file mode 100644 index 00000000..83659632 --- /dev/null +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveDoubleType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.hive.types; + +import com.linkedin.transport.api.types.StdDoubleType; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; + + +public class HiveDoubleType implements StdDoubleType { + + private final DoubleObjectInspector _doubleObjectInspector; + + public HiveDoubleType(DoubleObjectInspector doubleObjectInspector) { + _doubleObjectInspector = doubleObjectInspector; + } + + @Override + public Object underlyingType() { + return _doubleObjectInspector; + } +} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveFloatType.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveFloatType.java new file mode 100644 index 00000000..9f9107f6 --- /dev/null +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveFloatType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.hive.types; + +import com.linkedin.transport.api.types.StdFloatType; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; + + +public class HiveFloatType implements StdFloatType { + + private final FloatObjectInspector _floatObjectInspector; + + public HiveFloatType(FloatObjectInspector floatObjectInspector) { + _floatObjectInspector = floatObjectInspector; + } + + @Override + public Object underlyingType() { + return _floatObjectInspector; + } +} diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveStructType.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveRowType.java similarity index 83% rename from transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveStructType.java rename to transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveRowType.java index f4393776..c9ceef43 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveStructType.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/types/HiveRowType.java @@ -5,7 +5,7 @@ */ package com.linkedin.transport.hive.types; -import com.linkedin.transport.api.types.StdStructType; +import com.linkedin.transport.api.types.RowType; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.hive.HiveWrapper; import java.util.List; @@ -13,11 +13,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -public class HiveStructType implements StdStructType { +public class HiveRowType implements RowType { final StructObjectInspector _structObjectInspector; - public HiveStructType(StructObjectInspector structObjectInspector) { + public HiveRowType(StructObjectInspector structObjectInspector) { _structObjectInspector = structObjectInspector; } diff --git a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/typesystem/HiveTypeSystem.java b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/typesystem/HiveTypeSystem.java index 977779f1..4fa3f596 100644 --- a/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/typesystem/HiveTypeSystem.java +++ b/transportable-udfs-hive/src/main/java/com/linkedin/transport/hive/typesystem/HiveTypeSystem.java @@ -14,7 +14,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -69,6 +72,21 @@ protected boolean isStringType(ObjectInspector dataType) { return dataType instanceof StringObjectInspector; } + @Override + protected boolean isFloatType(ObjectInspector dataType) { + return dataType instanceof FloatObjectInspector; + } + + @Override + protected boolean isDoubleType(ObjectInspector dataType) { + return dataType instanceof DoubleObjectInspector; + } + + @Override + protected boolean isBinaryType(ObjectInspector dataType) { + return dataType instanceof BinaryObjectInspector; + } + @Override protected boolean isArrayType(ObjectInspector dataType) { return dataType instanceof ListObjectInspector; @@ -104,6 +122,21 @@ protected ObjectInspector createStringType() { return PrimitiveObjectInspectorFactory.javaStringObjectInspector; } + @Override + protected ObjectInspector createFloatType() { + return PrimitiveObjectInspectorFactory.javaFloatObjectInspector; + } + + @Override + protected ObjectInspector createDoubleType() { + return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector; + } + + @Override + protected ObjectInspector createBinaryType() { + return PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector; + } + @Override protected ObjectInspector createUnknownType() { return PrimitiveObjectInspectorFactory.javaVoidObjectInspector; diff --git a/transportable-udfs-plugin/build.gradle b/transportable-udfs-plugin/build.gradle index f465e780..6f4ade36 100644 --- a/transportable-udfs-plugin/build.gradle +++ b/transportable-udfs-plugin/build.gradle @@ -27,7 +27,7 @@ def writeVersionInfo = { file -> ant.propertyfile(file: file) { entry(key: "transport-version", value: version) entry(key: "hive-version", value: '1.2.2') - entry(key: "presto-version", value: '319') + entry(key: "presto-version", value: '333') entry(key: "spark-version", value: '2.3.0') entry(key: "scala-version", value: '2.11.8') } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java index ee96e18d..64a967f4 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java @@ -11,6 +11,7 @@ import com.linkedin.transport.codegen.SparkWrapperGenerator; import com.linkedin.transport.plugin.packaging.DistributionPackaging; import com.linkedin.transport.plugin.packaging.ShadedJarPackaging; +import com.linkedin.transport.plugin.packaging.ThinJarPackaging; import java.io.IOException; import java.io.InputStream; import java.util.List; @@ -72,7 +73,7 @@ private static Properties loadDefaultVersions() { // converters drop dependencies with classifiers, so we apply this dependency explicitly getDependencyConfiguration(RUNTIME_ONLY, "io.prestosql:presto-main", "presto", "tests") ), - new DistributionPackaging()), + ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())), new Platform( "hive", Language.JAVA, @@ -85,7 +86,7 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", "transport") ), - new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null)), + ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))), new Platform( "spark", Language.SCALA, @@ -99,9 +100,9 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-spark", "transport") ), - new ShadedJarPackaging( + ImmutableList.of(new ShadedJarPackaging( ImmutableList.of("org.apache.hadoop", "org.apache.spark"), - ImmutableList.of("com.linkedin.transport.spark.**")) + ImmutableList.of("com.linkedin.transport.spark.**"))) ) ); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java index fed24365..b3d87679 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java @@ -20,11 +20,11 @@ public class Platform { private final Class _wrapperGeneratorClass; private final List _defaultWrapperDependencyConfigurations; private final List _defaultTestDependencyConfigurations; - private final Packaging _packaging; + private final List _packaging; public Platform(String name, Language language, Class wrapperGeneratorClass, List defaultWrapperDependencyConfigurations, - List defaultTestDependencyConfigurations, Packaging packaging) { + List defaultTestDependencyConfigurations, List packaging) { _name = name; _language = language; _wrapperGeneratorClass = wrapperGeneratorClass; @@ -53,7 +53,7 @@ public List getDefaultTestDependencyConfigurations() { return _defaultTestDependencyConfigurations; } - public Packaging getPackaging() { + public List getPackaging() { return _packaging; } } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 74958b6d..2f8e2984 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -12,6 +12,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.stream.Collectors; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -25,6 +26,8 @@ import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; import org.gradle.language.base.plugins.LifecycleBasePlugin; +import org.gradle.testing.jacoco.plugins.JacocoPlugin; +import org.gradle.testing.jacoco.plugins.JacocoTaskExtension; import static com.linkedin.transport.plugin.ConfigurationType.*; import static com.linkedin.transport.plugin.SourceSetUtils.*; @@ -45,18 +48,32 @@ public class TransportPlugin implements Plugin { public void apply(Project project) { + + TransportPluginConfig extension = project.getExtensions().create("transport", TransportPluginConfig.class, project); + project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { project.getPlugins().apply(ScalaPlugin.class); project.getPlugins().apply(DistributionPlugin.class); project.getConfigurations().create(ShadowBasePlugin.getCONFIGURATION_NAME()); JavaPluginConvention javaConvention = project.getConvention().getPlugin(JavaPluginConvention.class); - SourceSet mainSourceSet = javaConvention.getSourceSets().getByName("main"); - SourceSet testSourceSet = javaConvention.getSourceSets().getByName("test"); + SourceSet mainSourceSet = javaConvention.getSourceSets().getByName(extension.mainSourceSetName); + SourceSet testSourceSet = javaConvention.getSourceSets().getByName(extension.testSourceSetName); configureBaseSourceSets(project, mainSourceSet, testSourceSet); Defaults.DEFAULT_PLATFORMS.forEach( - platform -> configurePlatform(project, platform, mainSourceSet, testSourceSet)); + platform -> configurePlatform(project, platform, mainSourceSet, testSourceSet, extension.outputDirFile)); + }); + // Disable Jacoco for platform test tasks as it is known to cause issues with Presto and Hive tests + project.getPlugins().withType(JacocoPlugin.class, (jacocoPlugin) -> { + Defaults.DEFAULT_PLATFORMS.forEach(platform -> { + project.getTasksByName(testTaskName(platform), true).forEach(task -> { + JacocoTaskExtension jacocoExtension = task.getExtensions().findByType(JacocoTaskExtension.class); + if (jacocoExtension != null) { + jacocoExtension.setEnabled(false); + } + }); + }); }); } @@ -77,8 +94,9 @@ private void configureBaseSourceSets(Project project, SourceSet mainSourceSet, S /** * Configures SourceSets, dependencies and tasks related to each Transport UDF platform */ - private void configurePlatform(Project project, Platform platform, SourceSet mainSourceSet, SourceSet testSourceSet) { - SourceSet sourceSet = configureSourceSet(project, platform, mainSourceSet); + private void configurePlatform(Project project, Platform platform, SourceSet mainSourceSet, SourceSet testSourceSet, + File baseOutputDir) { + SourceSet sourceSet = configureSourceSet(project, platform, mainSourceSet, baseOutputDir); configureGenerateWrappersTask(project, platform, mainSourceSet, sourceSet); List> packagingTasks = configurePackagingTasks(project, platform, sourceSet, mainSourceSet); @@ -94,9 +112,9 @@ private void configurePlatform(Project project, Platform platform, SourceSet mai * configurations and configures the default dependencies required for compilation and runtime of the wrapper * SourceSet */ - private SourceSet configureSourceSet(Project project, Platform platform, SourceSet mainSourceSet) { + private SourceSet configureSourceSet(Project project, Platform platform, SourceSet mainSourceSet, File baseOutputDir) { JavaPluginConvention javaConvention = project.getConvention().getPlugin(JavaPluginConvention.class); - Path platformBaseDir = Paths.get(project.getBuildDir().toString(), "generatedWrappers", platform.getName()); + Path platformBaseDir = Paths.get(baseOutputDir.toString(), "generatedWrappers", platform.getName()); Path wrapperSourceOutputDir = platformBaseDir.resolve("sources"); Path wrapperResourceOutputDir = platformBaseDir.resolve("resources"); @@ -186,7 +204,9 @@ private TaskProvider configureGenerateWrappersTask(Project */ private List> configurePackagingTasks(Project project, Platform platform, SourceSet sourceSet, SourceSet mainSourceSet) { - return platform.getPackaging().configurePackagingTasks(project, platform, sourceSet, mainSourceSet); + return platform.getPackaging().stream() + .flatMap(p -> p.configurePackagingTasks(project, platform, sourceSet, mainSourceSet).stream()) + .collect(Collectors.toList()); } /** @@ -230,7 +250,7 @@ task prestoTest(type: Test, dependsOn: test) { } */ - return project.getTasks().register(platform.getName() + "Test", Test.class, task -> { + return project.getTasks().register(testTaskName(platform), Test.class, task -> { task.setGroup(LifecycleBasePlugin.VERIFICATION_GROUP); task.setDescription("Runs Transport UDF tests on " + platform.getName()); task.setTestClassesDirs(testSourceSet.getOutput().getClassesDirs()); @@ -239,4 +259,8 @@ task prestoTest(type: Test, dependsOn: test) { task.mustRunAfter(project.getTasks().named("test")); }); } + + private String testTaskName(Platform platform) { + return platform.getName() + "Test"; + } } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java new file mode 100644 index 00000000..deff599a --- /dev/null +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.plugin; + +import java.io.File; +import org.gradle.api.Project; + + +/** + * Custom configuration for the {@link TransportPlugin}. + */ +public class TransportPluginConfig { + + /** + * Available gradle property names. + */ + private static final String MAIN_SOURCE_SET_NAME_PROP = "transport-plugin-main-source-set"; + private static final String TEST_SOURCE_SET_NAME_PROP = "transport-plugin-test-source-set"; + private static final String OUTPUT_DIR_PROP = "transport-plugin-output-dir"; + + /** + * The main source set used by the plugin. + */ + public String mainSourceSetName; + /** + * The test source set used by the plugin. + */ + public String testSourceSetName; + /** + * The output code-gen directory, relative the to the project directory. + */ + public File outputDirFile; + + /** + * Create a config object from the gradle {@link Project}. + * + * On creation, we attempt to populate config values using gradle properties or set to default values. + */ + public TransportPluginConfig(Project project) { + mainSourceSetName = getPropertyOrDefault(project, MAIN_SOURCE_SET_NAME_PROP, "main"); + testSourceSetName = getPropertyOrDefault(project, TEST_SOURCE_SET_NAME_PROP, "test"); + outputDirFile = project.hasProperty(OUTPUT_DIR_PROP) + ? project.file(project.property(OUTPUT_DIR_PROP).toString()) + : project.getBuildDir(); // Build dir by default + } + + /** + * Retrieve a string property from a gradle project if it has been set, a default value otherwise. + */ + private String getPropertyOrDefault(Project project, String propertyName, String defaultValue) { + return project.hasProperty(propertyName) + ? project.property(propertyName).toString() + : defaultValue; + } +} diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java index 44ef8b55..13de71db 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java @@ -66,18 +66,18 @@ public List> configurePackagingTasks(Project projec */ private TaskProvider createThinJarTask(Project project, SourceSet sourceSet, String platformName) { /* - task ThinJar(type: Jar, dependsOn: prestoClasses) { - classifier 'platformName' + task DistThinJar(type: Jar, dependsOn: prestoClasses) { + classifier '-dist-thin' from sourceSets..output from sourceSets..resources } */ - return project.getTasks().register(sourceSet.getTaskName(null, "thinJar"), Jar.class, task -> { + return project.getTasks().register(sourceSet.getTaskName(null, "distThinJar"), Jar.class, task -> { task.dependsOn(project.getTasks().named(sourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platformName + " classes to be included in the distribution"); - task.setClassifier(platformName + "Thin"); + task.setClassifier(platformName + "-dist-thin"); task.from(sourceSet.getOutput()); task.from(sourceSet.getResources()); }); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java new file mode 100644 index 00000000..7367733c --- /dev/null +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java @@ -0,0 +1,48 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.plugin.packaging; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.plugin.Platform; +import java.util.List; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.bundling.Jar; + + +/** + * A {@link Packaging} class which generates a Jar containing the autogenerated code for the platform + */ +public class ThinJarPackaging implements Packaging { + + @Override + public List> configurePackagingTasks(Project project, Platform platform, + SourceSet platformSourceSet, SourceSet mainSourceSet) { + /* + task ThinJar(type: Jar, dependsOn: prestoClasses) { + classifier '-thin' + from sourceSets..output + from sourceSets..resources + } + */ + + TaskProvider thinJarTask = + project.getTasks().register(platformSourceSet.getTaskName(null, "thinJar"), Jar.class, task -> { + task.dependsOn(project.getTasks().named(platformSourceSet.getClassesTaskName())); + task.setDescription("Assembles a thin jar archive containing the " + platform.getName() + + " classes to be included in the distribution"); + task.setClassifier(platform.getName() + "-thin"); + task.from(platformSourceSet.getOutput()); + task.from(platformSourceSet.getResources()); + }); + + project.getArtifacts().add(Dependency.ARCHIVES_CONFIGURATION, thinJarTask); + return ImmutableList.of(thinJarTask); + } +} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/FileSystemClient.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/FileSystemClient.java index 850ef304..f964433d 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/FileSystemClient.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/FileSystemClient.java @@ -53,7 +53,9 @@ public String copyToLocalFile(String remoteFilename) { Path remotePath = new Path(remoteFilename); Path localPath = new Path(Paths.get(getAndCreateLocalDir(), new File(remoteFilename).getName()).toString()); FileSystem fs = remotePath.getFileSystem(conf); - String resolvedRemoteFilename = FileSystemUtils.resolveLatest(remoteFilename, fs); + // It is important to pass the custom configuration object to FileSystemUtils since we load some extra + // properties from etc/**.xml in getConfiguration() for Presto + String resolvedRemoteFilename = FileSystemUtils.resolveLatest(remoteFilename, conf); Path resolvedRemotePath = new Path(resolvedRemoteFilename); fs.copyToLocalFile(resolvedRemotePath, localPath); return localPath.toString(); diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java index c5e0c53e..94539835 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java @@ -5,38 +5,31 @@ */ package com.linkedin.transport.presto; -import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdBoolean; -import com.linkedin.transport.api.data.StdInteger; -import com.linkedin.transport.api.data.StdLong; -import com.linkedin.transport.api.data.StdMap; -import com.linkedin.transport.api.data.StdString; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.presto.data.PrestoArray; -import com.linkedin.transport.presto.data.PrestoBoolean; -import com.linkedin.transport.presto.data.PrestoInteger; -import com.linkedin.transport.presto.data.PrestoLong; -import com.linkedin.transport.presto.data.PrestoMap; -import com.linkedin.transport.presto.data.PrestoString; -import com.linkedin.transport.presto.data.PrestoStruct; -import io.airlift.slice.Slices; +import com.linkedin.transport.presto.data.PrestoArrayData; +import com.linkedin.transport.presto.data.PrestoMapData; +import com.linkedin.transport.presto.data.PrestoRowData; import io.prestosql.metadata.BoundVariables; import io.prestosql.metadata.Metadata; -import io.prestosql.metadata.Signature; +import io.prestosql.metadata.OperatorNotFoundException; +import io.prestosql.metadata.ResolvedFunction; import io.prestosql.operator.scalar.ScalarFunctionImplementation; +import io.prestosql.spi.function.OperatorType; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.TypeSignature; +import java.nio.ByteBuffer; import java.util.List; import java.util.stream.Collectors; import static io.prestosql.metadata.SignatureBinder.*; - +import static io.prestosql.operator.TypeSignatureParser.*; public class PrestoFactory implements StdFactory { @@ -49,65 +42,48 @@ public PrestoFactory(BoundVariables boundVariables, Metadata metadata) { } @Override - public StdInteger createInteger(int value) { - return new PrestoInteger(value); - } - - @Override - public StdLong createLong(long value) { - return new PrestoLong(value); + public ArrayData createArray(StdType stdType, int expectedSize) { + return new PrestoArrayData((ArrayType) stdType.underlyingType(), expectedSize, this); } @Override - public StdBoolean createBoolean(boolean value) { - return new PrestoBoolean(value); - } - - @Override - public StdString createString(String value) { - Preconditions.checkNotNull(value, "Cannot create a null StdString"); - return new PrestoString(Slices.utf8Slice(value)); - } - - @Override - public StdArray createArray(StdType stdType, int expectedSize) { - return new PrestoArray((ArrayType) stdType.underlyingType(), expectedSize, this); - } - - @Override - public StdArray createArray(StdType stdType) { + public ArrayData createArray(StdType stdType) { return createArray(stdType, 0); } @Override - public StdMap createMap(StdType stdType) { - return new PrestoMap((MapType) stdType.underlyingType(), this); + public MapData createMap(StdType stdType) { + return new PrestoMapData((MapType) stdType.underlyingType(), this); } @Override - public PrestoStruct createStruct(List fieldNames, List fieldTypes) { - return new PrestoStruct(fieldNames, + public PrestoRowData createStruct(List fieldNames, List fieldTypes) { + return new PrestoRowData(fieldNames, fieldTypes.stream().map(stdType -> (Type) stdType.underlyingType()).collect(Collectors.toList()), this); } @Override - public PrestoStruct createStruct(List fieldTypes) { - return new PrestoStruct( + public PrestoRowData createStruct(List fieldTypes) { + return new PrestoRowData( fieldTypes.stream().map(stdType -> (Type) stdType.underlyingType()).collect(Collectors.toList()), this); } @Override - public StdStruct createStruct(StdType stdType) { - return new PrestoStruct((RowType) stdType.underlyingType(), this); + public RowData createStruct(StdType stdType) { + return new PrestoRowData((RowType) stdType.underlyingType(), this); } @Override public StdType createStdType(String typeSignature) { return PrestoWrapper.createStdType( - metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(typeSignature), boundVariables))); + metadata.getType(applyBoundVariables(parseTypeSignature(typeSignature, ImmutableSet.of()), boundVariables))); + } + + public ScalarFunctionImplementation getScalarFunctionImplementation(ResolvedFunction resolvedFunction) { + return metadata.getScalarFunctionImplementation(resolvedFunction); } - public ScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) { - return metadata.getScalarFunctionImplementation(signature); + public ResolvedFunction resolveOperator(OperatorType operatorType, List argumentTypes) throws OperatorNotFoundException { + return metadata.resolveOperator(operatorType, argumentTypes); } } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoWrapper.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoWrapper.java index 5dbf85f0..a1633ebb 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoWrapper.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoWrapper.java @@ -6,36 +6,52 @@ package com.linkedin.transport.presto; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.PlatformData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.presto.data.PrestoArray; -import com.linkedin.transport.presto.data.PrestoBoolean; -import com.linkedin.transport.presto.data.PrestoInteger; -import com.linkedin.transport.presto.data.PrestoLong; -import com.linkedin.transport.presto.data.PrestoMap; -import com.linkedin.transport.presto.data.PrestoString; -import com.linkedin.transport.presto.data.PrestoStruct; +import com.linkedin.transport.presto.data.PrestoArrayData; +import com.linkedin.transport.presto.data.PrestoData; +import com.linkedin.transport.presto.data.PrestoMapData; +import com.linkedin.transport.presto.data.PrestoRowData; import com.linkedin.transport.presto.types.PrestoArrayType; import com.linkedin.transport.presto.types.PrestoBooleanType; +import com.linkedin.transport.presto.types.PrestoBinaryType; +import com.linkedin.transport.presto.types.PrestoDoubleType; +import com.linkedin.transport.presto.types.PrestoFloatType; import com.linkedin.transport.presto.types.PrestoIntegerType; import com.linkedin.transport.presto.types.PrestoLongType; import com.linkedin.transport.presto.types.PrestoMapType; import com.linkedin.transport.presto.types.PrestoStringType; -import com.linkedin.transport.presto.types.PrestoStructType; +import com.linkedin.transport.presto.types.PrestoRowType; import com.linkedin.transport.presto.types.PrestoUnknownType; import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RealType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; import io.prestosql.type.UnknownType; +import java.nio.ByteBuffer; +import static io.prestosql.spi.type.BigintType.*; +import static io.prestosql.spi.type.BooleanType.*; +import static io.prestosql.spi.type.DoubleType.*; +import static io.prestosql.spi.type.IntegerType.*; +import static io.prestosql.spi.type.VarbinaryType.*; +import static io.prestosql.spi.type.VarcharType.*; +import static io.prestosql.spi.StandardErrorCode.*; +import static java.lang.Float.*; import static java.lang.Math.*; +import static java.lang.String.*; public final class PrestoWrapper { @@ -43,32 +59,94 @@ public final class PrestoWrapper { private PrestoWrapper() { } - public static StdData createStdData(Object prestoData, Type prestoType, StdFactory stdFactory) { + public static Object createStdData(Object prestoData, Type prestoType, StdFactory stdFactory) { if (prestoData == null) { return null; } if (prestoType instanceof IntegerType) { // Presto represents SQL Integers (i.e., corresponding to IntegerType above) as long or Long - // Therefore, to pass it to the PrestoInteger class, we first cast it to Long, then extract - // the int value. - return new PrestoInteger(((Long) prestoData).intValue()); - } else if (prestoType instanceof BigintType) { - return new PrestoLong((long) prestoData); - } else if (prestoType.getJavaType() == boolean.class) { - return new PrestoBoolean((boolean) prestoData); - } else if (prestoType.getJavaType() == Slice.class) { - return new PrestoString((Slice) prestoData); + // Therefore, we first cast prestoData to Long, then extract the int value. + return ((Long) prestoData).intValue(); + } else if (prestoType instanceof BigintType || prestoType.getJavaType() == boolean.class + || prestoType instanceof DoubleType) { + return prestoData; + } else if (prestoType instanceof VarcharType) { + return ((Slice) prestoData).toStringUtf8(); + } else if (prestoType instanceof RealType) { + // Presto represents SQL Reals (i.e., corresponding to RealType above) as long or Long + // Therefore, to pass it to the PrestoFloat class, we first cast it to Long, extract + // the int value and convert it the int bits to float. + long value = (long) prestoData; + int floatValue; + try { + floatValue = toIntExact(value); + } catch (ArithmeticException e) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, + format("Value (%sb) is not a valid single-precision float", Long.toBinaryString(value))); + } + return intBitsToFloat(floatValue); + } else if (prestoType instanceof VarbinaryType) { + return ((Slice) prestoData).toByteBuffer(); } else if (prestoType instanceof ArrayType) { - return new PrestoArray((Block) prestoData, (ArrayType) prestoType, stdFactory); + return new PrestoArrayData((Block) prestoData, (ArrayType) prestoType, stdFactory); } else if (prestoType instanceof MapType) { - return new PrestoMap((Block) prestoData, prestoType, stdFactory); + return new PrestoMapData((Block) prestoData, prestoType, stdFactory); } else if (prestoType instanceof RowType) { - return new PrestoStruct((Block) prestoData, prestoType, stdFactory); + return new PrestoRowData((Block) prestoData, prestoType, stdFactory); } assert false : "Unrecognized Presto Type: " + prestoType.getClass(); return null; } + public static Object getPlatformData(Object transportData) { + if (transportData == null) { + return null; + } + if (transportData instanceof Integer) { + return ((Number) transportData).longValue(); + } else if (transportData instanceof Long) { + return ((Long) transportData).longValue(); + } else if (transportData instanceof Float) { + return (long) floatToIntBits((Float) transportData); + } else if (transportData instanceof Double) { + return ((Double) transportData).doubleValue(); + } else if (transportData instanceof Boolean) { + return ((Boolean) transportData).booleanValue(); + } else if (transportData instanceof String) { + return Slices.utf8Slice((String) transportData); + } else if (transportData instanceof ByteBuffer) { + return Slices.wrappedBuffer(((ByteBuffer) transportData).array()); + } else { + return ((PlatformData) transportData).getUnderlyingData(); + } + } + + public static void writeToBlock(Object transportData, BlockBuilder blockBuilder) { + if (transportData == null) { + blockBuilder.appendNull(); + } else { + if (transportData instanceof Integer) { + // This looks a bit strange, but the call to writeLong is correct here. INTEGER does not have a writeInt method for + // some reason. It uses BlockBuilder.writeInt internally. + INTEGER.writeLong(blockBuilder, (Integer) transportData); + } else if (transportData instanceof Long) { + BIGINT.writeLong(blockBuilder, (Long) transportData); + } else if (transportData instanceof Float) { + INTEGER.writeLong(blockBuilder, floatToIntBits((Float) transportData)); + } else if (transportData instanceof Double) { + DOUBLE.writeDouble(blockBuilder, (Double) transportData); + } else if (transportData instanceof Boolean) { + BOOLEAN.writeBoolean(blockBuilder, (Boolean) transportData); + } else if (transportData instanceof String) { + VARCHAR.writeSlice(blockBuilder, Slices.utf8Slice((String) transportData)); + } else if (transportData instanceof ByteBuffer) { + VARBINARY.writeSlice(blockBuilder, Slices.wrappedBuffer((ByteBuffer) transportData)); + } else { + ((PrestoData) transportData).writeToBlock(blockBuilder); + } + } + } + public static StdType createStdType(Object prestoType) { if (prestoType instanceof IntegerType) { return new PrestoIntegerType((IntegerType) prestoType); @@ -78,12 +156,18 @@ public static StdType createStdType(Object prestoType) { return new PrestoBooleanType((BooleanType) prestoType); } else if (prestoType instanceof VarcharType) { return new PrestoStringType((VarcharType) prestoType); + } else if (prestoType instanceof RealType) { + return new PrestoFloatType((RealType) prestoType); + } else if (prestoType instanceof DoubleType) { + return new PrestoDoubleType((DoubleType) prestoType); + } else if (prestoType instanceof VarbinaryType) { + return new PrestoBinaryType((VarbinaryType) prestoType); } else if (prestoType instanceof ArrayType) { return new PrestoArrayType((ArrayType) prestoType); } else if (prestoType instanceof MapType) { return new PrestoMapType((MapType) prestoType); } else if (prestoType instanceof RowType) { - return new PrestoStructType(((RowType) prestoType)); + return new PrestoRowType(((RowType) prestoType)); } else if (prestoType instanceof UnknownType) { return new PrestoUnknownType(((UnknownType) prestoType)); } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java index f666e9d1..04628009 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java @@ -7,9 +7,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Booleans; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.StdUDF1; @@ -23,7 +23,9 @@ import com.linkedin.transport.api.udf.TopLevelStdUDF; import com.linkedin.transport.typesystem.GenericTypeSignatureElement; import io.prestosql.metadata.BoundVariables; +import io.prestosql.metadata.FunctionArgumentDefinition; import io.prestosql.metadata.FunctionKind; +import io.prestosql.metadata.FunctionMetadata; import io.prestosql.metadata.Metadata; import io.prestosql.metadata.Signature; import io.prestosql.metadata.SqlScalarFunction; @@ -32,7 +34,6 @@ import io.prestosql.spi.classloader.ThreadContextClassLoader; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.TypeSignature; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; @@ -41,13 +42,14 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.lang3.ClassUtils; import static io.prestosql.metadata.Signature.*; import static io.prestosql.metadata.SignatureBinder.*; -import static io.prestosql.spi.type.TypeSignature.*; +import static io.prestosql.operator.TypeSignatureParser.parseTypeSignature; import static io.prestosql.util.Reflection.*; // Suppressing argument naming convention for the evalInternal methods @@ -56,17 +58,26 @@ public abstract class StdUdfWrapper extends SqlScalarFunction { private static final int DEFAULT_REFRESH_INTERVAL_DAYS = 1; private static final int JITTER_FACTOR = 50; // to calculate jitter from delay - private volatile long _requiredFilesNextRefreshTime = Long.MAX_VALUE; // Will be set in specialize() - private String _functionDescription; protected StdUdfWrapper(StdUDF stdUDF) { - super(new Signature(((TopLevelStdUDF) stdUDF).getFunctionName(), FunctionKind.SCALAR, - getTypeVariableConstraintsForStdUdf(stdUDF), ImmutableList.of(), - parseTypeSignature(stdUDF.getOutputParameterSignature()), stdUDF.getInputParameterSignatures() - .stream() - .map(TypeSignature::parseTypeSignature) - .collect(Collectors.toList()), false)); - _functionDescription = ((TopLevelStdUDF) stdUDF).getFunctionDescription(); + super(new FunctionMetadata( + new Signature( + ((TopLevelStdUDF) stdUDF).getFunctionName(), + getTypeVariableConstraintsForStdUdf(stdUDF), + ImmutableList.of(), + parseTypeSignature(stdUDF.getOutputParameterSignature(), ImmutableSet.of()), + stdUDF.getInputParameterSignatures().stream() + .map(typeSignature -> parseTypeSignature(typeSignature, ImmutableSet.of())) + .collect(Collectors.toList()), + false), + true, + Booleans.asList(stdUDF.getNullableArguments()).stream() + .map(FunctionArgumentDefinition::new) + .collect(Collectors.toList()), + false, + false, + ((TopLevelStdUDF) stdUDF).getFunctionDescription(), + FunctionKind.SCALAR)); } @VisibleForTesting @@ -84,40 +95,25 @@ protected long getRefreshIntervalMillis() { return TimeUnit.DAYS.toMillis(DEFAULT_REFRESH_INTERVAL_DAYS); } - @Override - public boolean isHidden() { - return false; - } - - @Override - public boolean isDeterministic() { - return false; - } - - @Override - public String getDescription() { - return _functionDescription; - } - @Override public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, Metadata metadata) { StdFactory stdFactory = new PrestoFactory(boundVariables, metadata); StdUDF stdUDF = getStdUDF(); stdUDF.init(stdFactory); // Subtract a small jitter value so that refresh is triggered on first call - // Do not add extra delay, if refresh time was set to lower value by an earlier specialize + // while ensuring subsequent calls do not happen at the same time across workers long initialJitter = getRefreshIntervalMillis() / JITTER_FACTOR; int initialJitterInt = initialJitter > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) initialJitter; - _requiredFilesNextRefreshTime = - Math.min(_requiredFilesNextRefreshTime, System.currentTimeMillis() - (new Random()).nextInt(initialJitterInt)); + AtomicLong requiredFilesNextRefreshTime = new AtomicLong(System.currentTimeMillis() + - (new Random()).nextInt(initialJitterInt)); boolean[] nullableArguments = stdUDF.getAndCheckNullableArguments(); return new ScalarFunctionImplementation(true, getNullConventionForArguments(nullableArguments), - getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments), isDeterministic()); + getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments, requiredFilesNextRefreshTime)); } private MethodHandle getMethodHandle(StdUDF stdUDF, Metadata metadata, BoundVariables boundVariables, - boolean[] nullableArguments) { + boolean[] nullableArguments, AtomicLong requiredFilesNextRefreshTime) { Type[] inputTypes = getPrestoTypes(stdUDF.getInputParameterSignatures(), metadata, boundVariables); Type outputType = getPrestoType(stdUDF.getOutputParameterSignature(), metadata, boundVariables); @@ -134,7 +130,7 @@ private MethodHandle getMethodHandle(StdUDF stdUDF, Metadata metadata, BoundVari // Specific MethodHandle required by presto where argument types map to the type signature MethodHandle specificMethodHandle = MethodHandles.explicitCastArguments(genericMethodHandle, specificMethodType); return MethodHandles.insertArguments(specificMethodHandle, 0, stdUDF, inputTypes, - outputType instanceof IntegerType); + outputType instanceof IntegerType, requiredFilesNextRefreshTime); } private List getNullConventionForArguments( @@ -146,9 +142,9 @@ private List getNullConventionFor .collect(Collectors.toList()); } - private StdData[] wrapArguments(StdUDF stdUDF, Type[] types, Object[] arguments) { + private Object[] wrapArguments(StdUDF stdUDF, Type[] types, Object[] arguments) { StdFactory stdFactory = stdUDF.getStdFactory(); - StdData[] stdData = new StdData[arguments.length]; + Object[] stdData = new Object[arguments.length]; // TODO: Reuse wrapper objects by creating them once upon initialization and reuse them here // along the same lines of what we do in Hive implementation. // JIRA: https://jira01.corp.linkedin.com:8443/browse/LIHADOOP-34894 @@ -158,13 +154,14 @@ private StdData[] wrapArguments(StdUDF stdUDF, Type[] types, Object[] arguments) return stdData; } - protected Object eval(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object... arguments) { - StdData[] args = wrapArguments(stdUDF, types, arguments); - if (_requiredFilesNextRefreshTime < System.currentTimeMillis()) { + protected Object eval(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object... arguments) { + Object[] args = wrapArguments(stdUDF, types, arguments); + if (requiredFilesNextRefreshTime.get() <= System.currentTimeMillis()) { String[] requiredFiles = getRequiredFiles(stdUDF, args); - processRequiredFiles(stdUDF, requiredFiles); + processRequiredFiles(stdUDF, requiredFiles, requiredFilesNextRefreshTime); } - StdData result; + Object result; switch (args.length) { case 0: result = ((StdUDF0) stdUDF).eval(); @@ -196,16 +193,11 @@ protected Object eval(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, default: throw new RuntimeException("eval not supported yet for StdUDF" + args.length); } - if (result == null) { - return null; - } else if (isIntegerReturnType) { - return ((Number) ((PlatformData) result).getUnderlyingData()).longValue(); - } else { - return ((PlatformData) result).getUnderlyingData(); - } + + return PrestoWrapper.getPlatformData(result); } - private String[] getRequiredFiles(StdUDF stdUDF, StdData[] args) { + private String[] getRequiredFiles(StdUDF stdUDF, Object[] args) { String[] requiredFiles; switch (args.length) { case 0: @@ -243,8 +235,9 @@ private String[] getRequiredFiles(StdUDF stdUDF, StdData[] args) { return requiredFiles; } - private synchronized void processRequiredFiles(StdUDF stdUDF, String[] requiredFiles) { - if (_requiredFilesNextRefreshTime < System.currentTimeMillis()) { + private synchronized void processRequiredFiles(StdUDF stdUDF, String[] requiredFiles, + AtomicLong requiredFilesNextRefreshTime) { + if (requiredFilesNextRefreshTime.get() <= System.currentTimeMillis()) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(getClass().getClassLoader())) { String[] copiedFiles = new String[requiredFiles.length]; FileSystemClient client = new FileSystemClient(); @@ -255,8 +248,8 @@ private synchronized void processRequiredFiles(StdUDF stdUDF, String[] requiredF stdUDF.processRequiredFiles(copiedFiles); // Determine how many times _refreshIntervalMillis needs to be added to go above currentTimeMillis int refreshIntervalFactor = (int) Math.ceil( - (System.currentTimeMillis() - _requiredFilesNextRefreshTime) / (double) getRefreshIntervalMillis()); - _requiredFilesNextRefreshTime += getRefreshIntervalMillis() * Math.max(1, refreshIntervalFactor); + (System.currentTimeMillis() - requiredFilesNextRefreshTime.get()) / (double) getRefreshIntervalMillis()); + requiredFilesNextRefreshTime.getAndAdd(getRefreshIntervalMillis() * Math.max(1, refreshIntervalFactor)); } } } @@ -274,20 +267,21 @@ private Type[] getPrestoTypes(List parameterSignatures, Metadata metadat } private Type getPrestoType(String parameterSignature, Metadata metadata, BoundVariables boundVariables) { - return metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(parameterSignature), boundVariables)); + return metadata.getType(applyBoundVariables(parseTypeSignature(parameterSignature, ImmutableSet.of()), boundVariables)); } private Class[] getMethodHandleArgumentTypes(Type[] argTypes, boolean[] nullableArguments, boolean useObjectForArgumentType) { - Class[] methodHandleArgumentTypes = new Class[argTypes.length + 3]; + Class[] methodHandleArgumentTypes = new Class[argTypes.length + 4]; methodHandleArgumentTypes[0] = StdUDF.class; methodHandleArgumentTypes[1] = Type[].class; methodHandleArgumentTypes[2] = boolean.class; + methodHandleArgumentTypes[3] = AtomicLong.class; for (int i = 0; i < argTypes.length; i++) { if (useObjectForArgumentType) { - methodHandleArgumentTypes[i + 3] = Object.class; + methodHandleArgumentTypes[i + 4] = Object.class; } else { - methodHandleArgumentTypes[i + 3] = getJavaTypeForNullability(argTypes[i], nullableArguments[i]); + methodHandleArgumentTypes[i + 4] = getJavaTypeForNullability(argTypes[i], nullableArguments[i]); } } return methodHandleArgumentTypes; @@ -295,45 +289,53 @@ private Class[] getMethodHandleArgumentTypes(Type[] argTypes, boolean[] nulla protected abstract StdUDF getStdUDF(); - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType) { - return eval(stdUDF, types, isIntegerReturnType); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1) { - return eval(stdUDF, types, isIntegerReturnType, arg1); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3, Object arg4) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3, arg4); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3, Object arg4) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3, arg4); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3, Object arg4, Object arg5) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3, arg4, arg5); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3, arg4, arg5); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3, Object arg4, Object arg5, Object arg6) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3, arg4, arg5, arg6); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5, + Object arg6) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3, arg4, arg5, arg6); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3, Object arg4, Object arg5, Object arg6, Object arg7) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3, arg4, arg5, arg6, arg7); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5, + Object arg6, Object arg7) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3, arg4, arg5, arg6, + arg7); } - public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, Object arg1, Object arg2, - Object arg3, Object arg4, Object arg5, Object arg6, Object arg7, Object arg8) { - return eval(stdUDF, types, isIntegerReturnType, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); + public Object evalInternal(StdUDF stdUDF, Type[] types, boolean isIntegerReturnType, + AtomicLong requiredFilesNextRefreshTime, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5, + Object arg6, Object arg7, Object arg8) { + return eval(stdUDF, types, isIntegerReturnType, requiredFilesNextRefreshTime, arg1, arg2, arg3, arg4, arg5, arg6, + arg7, arg8); } } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArray.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArrayData.java similarity index 76% rename from transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArray.java rename to transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArrayData.java index 41759716..c775ea6b 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArray.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoArrayData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.presto.data; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.presto.PrestoWrapper; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; @@ -19,7 +18,7 @@ import static io.prestosql.spi.type.TypeUtils.*; -public class PrestoArray extends PrestoData implements StdArray { +public class PrestoArrayData extends PrestoData implements ArrayData { private final StdFactory _stdFactory; private final ArrayType _arrayType; @@ -28,14 +27,14 @@ public class PrestoArray extends PrestoData implements StdArray { private Block _block; private BlockBuilder _mutable; - public PrestoArray(Block block, ArrayType arrayType, StdFactory stdFactory) { + public PrestoArrayData(Block block, ArrayType arrayType, StdFactory stdFactory) { _block = block; _arrayType = arrayType; _elementType = arrayType.getElementType(); _stdFactory = stdFactory; } - public PrestoArray(ArrayType arrayType, int expectedEntries, StdFactory stdFactory) { + public PrestoArrayData(ArrayType arrayType, int expectedEntries, StdFactory stdFactory) { _block = null; _elementType = arrayType.getElementType(); _mutable = _elementType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), expectedEntries); @@ -49,19 +48,19 @@ public int size() { } @Override - public StdData get(int idx) { + public E get(int idx) { Block sourceBlock = _mutable == null ? _block : _mutable; int position = PrestoWrapper.checkedIndexToBlockPosition(sourceBlock, idx); Object element = readNativeValue(_elementType, sourceBlock, position); - return PrestoWrapper.createStdData(element, _elementType, _stdFactory); + return (E) PrestoWrapper.createStdData(element, _elementType, _stdFactory); } @Override - public void add(StdData e) { + public void add(E e) { if (_mutable == null) { _mutable = _elementType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); } - ((PrestoData) e).writeToBlock(_mutable); + PrestoWrapper.writeToBlock(e, _mutable); } @Override @@ -75,10 +74,10 @@ public void setUnderlyingData(Object value) { } @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Block sourceBlock = _mutable == null ? _block : _mutable; - int size = PrestoArray.this.size(); + int size = PrestoArrayData.this.size(); int position = 0; @Override @@ -87,10 +86,10 @@ public boolean hasNext() { } @Override - public StdData next() { + public E next() { Object element = readNativeValue(_elementType, sourceBlock, position); position++; - return PrestoWrapper.createStdData(element, _elementType, _stdFactory); + return (E) PrestoWrapper.createStdData(element, _elementType, _stdFactory); } }; } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoBoolean.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoBoolean.java deleted file mode 100644 index 408fc9be..00000000 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoBoolean.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.presto.data; - -import com.linkedin.transport.api.data.StdBoolean; -import io.prestosql.spi.block.BlockBuilder; - -import static io.prestosql.spi.type.BooleanType.*; - - -public class PrestoBoolean extends PrestoData implements StdBoolean { - - boolean _value; - - public PrestoBoolean(boolean value) { - _value = value; - } - - @Override - public boolean get() { - return _value; - } - - @Override - public Object getUnderlyingData() { - return _value; - } - - @Override - public void setUnderlyingData(Object value) { - _value = (boolean) value; - } - - @Override - public void writeToBlock(BlockBuilder blockBuilder) { - BOOLEAN.writeBoolean(blockBuilder, _value); - } -} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoInteger.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoInteger.java deleted file mode 100644 index 06ef9a3b..00000000 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoInteger.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.presto.data; - -import com.linkedin.transport.api.data.StdInteger; -import io.prestosql.spi.block.BlockBuilder; - -import static io.prestosql.spi.type.IntegerType.*; - - -public class PrestoInteger extends PrestoData implements StdInteger { - - int _integer; - - public PrestoInteger(int integer) { - _integer = integer; - } - - @Override - public int get() { - return _integer; - } - - @Override - public Object getUnderlyingData() { - return _integer; - } - - @Override - public void setUnderlyingData(Object value) { - _integer = ((Long) value).intValue(); - } - - @Override - public void writeToBlock(BlockBuilder blockBuilder) { - // It looks a bit strange, but the call to writeLong is correct here. INTEGER does not have a writeInt method for - // some reason. It uses BlockBuilder.writeInt internally. - INTEGER.writeLong(blockBuilder, _integer); - } -} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoLong.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoLong.java deleted file mode 100644 index 29832b4a..00000000 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoLong.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.presto.data; - -import com.linkedin.transport.api.data.StdLong; -import io.prestosql.spi.block.BlockBuilder; - -import static io.prestosql.spi.type.BigintType.*; - - -public class PrestoLong extends PrestoData implements StdLong { - - long _value; - - public PrestoLong(long value) { - _value = value; - } - - @Override - public long get() { - return _value; - } - - @Override - public Object getUnderlyingData() { - return _value; - } - - @Override - public void setUnderlyingData(Object value) { - _value = (long) value; - } - - @Override - public void writeToBlock(BlockBuilder blockBuilder) { - BIGINT.writeLong(blockBuilder, _value); - } -} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMapData.java similarity index 71% rename from transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java rename to transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMapData.java index 4337819d..aecbf907 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMapData.java @@ -9,8 +9,7 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.presto.PrestoFactory; import com.linkedin.transport.presto.PrestoWrapper; import io.prestosql.spi.PrestoException; @@ -18,7 +17,6 @@ import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.block.PageBuilderStatus; import io.prestosql.spi.function.OperatorType; -import io.prestosql.spi.type.BooleanType; import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.Type; import java.lang.invoke.MethodHandle; @@ -28,12 +26,11 @@ import java.util.Iterator; import java.util.Set; -import static io.prestosql.metadata.Signature.*; import static io.prestosql.spi.StandardErrorCode.*; import static io.prestosql.spi.type.TypeUtils.*; -public class PrestoMap extends PrestoData implements StdMap { +public class PrestoMapData extends PrestoData implements MapData { final Type _keyType; final Type _valueType; @@ -42,7 +39,7 @@ public class PrestoMap extends PrestoData implements StdMap { final StdFactory _stdFactory; Block _block; - public PrestoMap(Type mapType, StdFactory stdFactory) { + public PrestoMapData(Type mapType, StdFactory stdFactory) { BlockBuilder mutable = mapType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); mutable.beginBlockEntry(); mutable.closeEntry(); @@ -54,11 +51,11 @@ public PrestoMap(Type mapType, StdFactory stdFactory) { _stdFactory = stdFactory; _keyEqualsMethod = ((PrestoFactory) stdFactory).getScalarFunctionImplementation( - internalOperator(OperatorType.EQUAL, BooleanType.BOOLEAN, ImmutableList.of(_keyType, _keyType))) + ((PrestoFactory) stdFactory).resolveOperator(OperatorType.EQUAL, ImmutableList.of(_keyType, _keyType))) .getMethodHandle(); } - public PrestoMap(Block block, Type mapType, StdFactory stdFactory) { + public PrestoMapData(Block block, Type mapType, StdFactory stdFactory) { this(mapType, stdFactory); _block = block; } @@ -69,13 +66,12 @@ public int size() { } @Override - public StdData get(StdData key) { - Object prestoKey = ((PlatformData) key).getUnderlyingData(); + public V get(K key) { + Object prestoKey = PrestoWrapper.getPlatformData(key); int i = seekKey(prestoKey); if (i != -1) { Object value = readNativeValue(_valueType, _block, i); - StdData stdValue = PrestoWrapper.createStdData(value, _valueType, _stdFactory); - return stdValue; + return (V) PrestoWrapper.createStdData(value, _valueType, _stdFactory); } else { return null; } @@ -84,10 +80,10 @@ public StdData get(StdData key) { // TODO: Do not copy the _mutable BlockBuilder on every update. As long as updates are append-only or for fixed-size // types, we can skip copying. @Override - public void put(StdData key, StdData value) { + public void put(K key, V value) { BlockBuilder mutable = _mapType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); BlockBuilder entryBuilder = mutable.beginBlockEntry(); - Object prestoKey = ((PlatformData) key).getUnderlyingData(); + Object prestoKey = PrestoWrapper.getPlatformData(key); int valuePosition = seekKey(prestoKey); for (int i = 0; i < _block.getPositionCount(); i += 2) { // Write the current key to the map @@ -95,26 +91,26 @@ public void put(StdData key, StdData value) { // Find out if we need to change the corresponding value if (i == valuePosition - 1) { // Use the user-supplied value - ((PrestoData) value).writeToBlock(entryBuilder); + PrestoWrapper.writeToBlock(value, entryBuilder); } else { // Use the existing value in original _block _valueType.appendTo(_block, i + 1, entryBuilder); } } if (valuePosition == -1) { - ((PrestoData) key).writeToBlock(entryBuilder); - ((PrestoData) value).writeToBlock(entryBuilder); + PrestoWrapper.writeToBlock(key, entryBuilder); + PrestoWrapper.writeToBlock(value, entryBuilder); } mutable.closeEntry(); _block = ((MapType) _mapType).getObject(mutable.build(), 0); } - public Set keySet() { - return new AbstractSet() { + public Set keySet() { + return new AbstractSet() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { int i = -2; @Override @@ -123,27 +119,27 @@ public boolean hasNext() { } @Override - public StdData next() { + public K next() { i += 2; - return PrestoWrapper.createStdData(readNativeValue(_keyType, _block, i), _keyType, _stdFactory); + return (K) PrestoWrapper.createStdData(readNativeValue(_keyType, _block, i), _keyType, _stdFactory); } }; } @Override public int size() { - return PrestoMap.this.size(); + return PrestoMapData.this.size(); } }; } @Override - public Collection values() { - return new AbstractCollection() { + public Collection values() { + return new AbstractCollection() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { int i = -2; @Override @@ -152,22 +148,25 @@ public boolean hasNext() { } @Override - public StdData next() { + public V next() { i += 2; - return PrestoWrapper.createStdData(readNativeValue(_valueType, _block, i + 1), _valueType, _stdFactory); + return + (V) PrestoWrapper.createStdData( + readNativeValue(_valueType, _block, i + 1), _valueType, _stdFactory + ); } }; } @Override public int size() { - return PrestoMap.this.size(); + return PrestoMapData.this.size(); } }; } @Override - public boolean containsKey(StdData key) { + public boolean containsKey(K key) { return get(key) != null; } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoStruct.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoRowData.java similarity index 83% rename from transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoStruct.java rename to transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoRowData.java index e48a94c4..20d56a09 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoStruct.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoRowData.java @@ -6,8 +6,7 @@ package com.linkedin.transport.presto.data; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.presto.PrestoWrapper; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; @@ -24,28 +23,28 @@ import static io.prestosql.spi.type.TypeUtils.*; -public class PrestoStruct extends PrestoData implements StdStruct { +public class PrestoRowData extends PrestoData implements RowData { final RowType _rowType; final StdFactory _stdFactory; Block _block; - public PrestoStruct(Type rowType, StdFactory stdFactory) { + public PrestoRowData(Type rowType, StdFactory stdFactory) { _rowType = (RowType) rowType; _stdFactory = stdFactory; } - public PrestoStruct(Block block, Type rowType, StdFactory stdFactory) { + public PrestoRowData(Block block, Type rowType, StdFactory stdFactory) { this(rowType, stdFactory); _block = block; } - public PrestoStruct(List fieldTypes, StdFactory stdFactory) { + public PrestoRowData(List fieldTypes, StdFactory stdFactory) { _stdFactory = stdFactory; _rowType = RowType.anonymous(fieldTypes); } - public PrestoStruct(List fieldNames, List fieldTypes, StdFactory stdFactory) { + public PrestoRowData(List fieldNames, List fieldTypes, StdFactory stdFactory) { _stdFactory = stdFactory; List fields = IntStream.range(0, fieldNames.size()) .mapToObj(i -> new RowType.Field(Optional.ofNullable(fieldNames.get(i)), fieldTypes.get(i))) @@ -54,7 +53,7 @@ public PrestoStruct(List fieldNames, List fieldTypes, StdFactory s } @Override - public StdData getField(int index) { + public Object getField(int index) { int position = PrestoWrapper.checkedIndexToBlockPosition(_block, index); if (position == -1) { return null; @@ -65,7 +64,7 @@ public StdData getField(int index) { } @Override - public StdData getField(String name) { + public Object getField(String name) { int index = -1; Type elementType = null; int i = 0; @@ -85,7 +84,7 @@ public StdData getField(String name) { } @Override - public void setField(int index, StdData value) { + public void setField(int index, Object value) { // TODO: This is not the right way to get this object. The status should be passed in from the invocation of the // function and propagated to here. See PRESTO-1359 for more details. BlockBuilderStatus blockBuilderStatus = new PageBuilderStatus().createBlockBuilderStatus(); @@ -94,7 +93,7 @@ public void setField(int index, StdData value) { int i = 0; for (RowType.Field field : _rowType.getFields()) { if (i == index) { - ((PrestoData) value).writeToBlock(rowBlockBuilder); + PrestoWrapper.writeToBlock(value, rowBlockBuilder); } else { if (_block == null) { rowBlockBuilder.appendNull(); @@ -109,13 +108,13 @@ public void setField(int index, StdData value) { } @Override - public void setField(String name, StdData value) { + public void setField(String name, Object value) { BlockBuilder mutable = _rowType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); BlockBuilder rowBlockBuilder = mutable.beginBlockEntry(); int i = 0; for (RowType.Field field : _rowType.getFields()) { if (field.getName().isPresent() && name.equals(field.getName().get())) { - ((PrestoData) value).writeToBlock(rowBlockBuilder); + PrestoWrapper.writeToBlock(value, rowBlockBuilder); } else { if (_block == null) { rowBlockBuilder.appendNull(); @@ -130,8 +129,8 @@ public void setField(String name, StdData value) { } @Override - public List fields() { - ArrayList fields = new ArrayList<>(); + public List fields() { + ArrayList fields = new ArrayList<>(); for (int i = 0; i < _block.getPositionCount(); i++) { Type elementType = _rowType.getFields().get(i).getType(); Object element = readNativeValue(elementType, _block, i); diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoString.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoString.java deleted file mode 100644 index 6691da3f..00000000 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoString.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.presto.data; - -import com.linkedin.transport.api.data.StdString; -import io.airlift.slice.Slice; -import io.prestosql.spi.block.BlockBuilder; - -import static io.prestosql.spi.type.VarcharType.*; - - -public class PrestoString extends PrestoData implements StdString { - - Slice _slice; - - public PrestoString(Slice slice) { - _slice = slice; - } - - @Override - public String get() { - return _slice.toStringUtf8(); - } - - @Override - public Object getUnderlyingData() { - return _slice; - } - - @Override - public void setUnderlyingData(Object value) { - _slice = (Slice) value; - } - - @Override - public void writeToBlock(BlockBuilder blockBuilder) { - VARCHAR.writeSlice(blockBuilder, _slice); - } -} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoBinaryType.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoBinaryType.java new file mode 100644 index 00000000..1be446f1 --- /dev/null +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoBinaryType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.presto.types; + +import com.linkedin.transport.api.types.StdBinaryType; +import io.prestosql.spi.type.VarbinaryType; + + +public class PrestoBinaryType implements StdBinaryType { + + private final VarbinaryType varbinaryType; + + public PrestoBinaryType(VarbinaryType varbinaryType) { + this.varbinaryType = varbinaryType; + } + + @Override + public Object underlyingType() { + return varbinaryType; + } +} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoDoubleType.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoDoubleType.java new file mode 100644 index 00000000..a9a6394e --- /dev/null +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoDoubleType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.presto.types; + +import com.linkedin.transport.api.types.StdDoubleType; +import io.prestosql.spi.type.DoubleType; + + +public class PrestoDoubleType implements StdDoubleType { + + private final DoubleType doubleType; + + public PrestoDoubleType(DoubleType doubleType) { + this.doubleType = doubleType; + } + + @Override + public Object underlyingType() { + return doubleType; + } +} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoFloatType.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoFloatType.java new file mode 100644 index 00000000..2b481c64 --- /dev/null +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoFloatType.java @@ -0,0 +1,24 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.presto.types; + +import com.linkedin.transport.api.types.StdFloatType; +import io.prestosql.spi.type.RealType; + + +public class PrestoFloatType implements StdFloatType { + + private final RealType floatType; + + public PrestoFloatType(RealType floatType) { + this.floatType = floatType; + } + + @Override + public Object underlyingType() { + return floatType; + } +} diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoLongType.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoLongType.java index 1eecf393..f0dbb856 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoLongType.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoLongType.java @@ -5,11 +5,11 @@ */ package com.linkedin.transport.presto.types; -import com.linkedin.transport.api.types.StdIntegerType; +import com.linkedin.transport.api.types.StdLongType; import io.prestosql.spi.type.BigintType; -public class PrestoLongType implements StdIntegerType { +public class PrestoLongType implements StdLongType { final BigintType bigintType; diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoStructType.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoRowType.java similarity index 75% rename from transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoStructType.java rename to transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoRowType.java index f94bd051..d372aab0 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoStructType.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/types/PrestoRowType.java @@ -5,19 +5,18 @@ */ package com.linkedin.transport.presto.types; -import com.linkedin.transport.api.types.StdStructType; +import com.linkedin.transport.api.types.RowType; import com.linkedin.transport.api.types.StdType; import com.linkedin.transport.presto.PrestoWrapper; -import io.prestosql.spi.type.RowType; import java.util.List; import java.util.stream.Collectors; -public class PrestoStructType implements StdStructType { +public class PrestoRowType implements RowType { - final RowType rowType; + final io.prestosql.spi.type.RowType rowType; - public PrestoStructType(RowType rowType) { + public PrestoRowType(io.prestosql.spi.type.RowType rowType) { this.rowType = rowType; } diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkFactory.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkFactory.scala index c3eeecce..87d5625d 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkFactory.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkFactory.scala @@ -5,51 +5,39 @@ */ package com.linkedin.transport.spark +import java.nio.ByteBuffer import java.util.{List => JavaList} -import com.google.common.base.Preconditions import com.linkedin.transport.api.StdFactory import com.linkedin.transport.api.data._ import com.linkedin.transport.api.types.StdType import com.linkedin.transport.spark.data._ import com.linkedin.transport.spark.typesystem.SparkTypeFactory import com.linkedin.transport.typesystem.{AbstractBoundVariables, TypeSignature} -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType} class SparkFactory(private val _boundVariables: AbstractBoundVariables[DataType]) extends StdFactory { private val _sparkTypeFactory: SparkTypeFactory = new SparkTypeFactory - override def createInteger(value: Int): StdInteger = SparkInteger(value) - - override def createLong(value: Long): StdLong = SparkLong(value) - - override def createBoolean(value: Boolean): StdBoolean = SparkBoolean(value) - - override def createString(value: String): StdString = { - Preconditions.checkNotNull(value, "Cannot create a null StdString".asInstanceOf[Any]) - SparkString(UTF8String.fromString(value)) - } - - override def createArray(stdType: StdType): StdArray = createArray(stdType, 0) + override def createArray(stdType: StdType): ArrayData[_] = createArray(stdType, 0) // we do not pass size to `new Array()` as the size argument of createArray is supposed to be just a hint about - // the expected number of entries in the StdArray. `new Array(size)` will create an array with null entries - override def createArray(stdType: StdType, size: Int): StdArray = SparkArray( + // the expected number of entries in the ArrayData. `new Array(size)` will create an array with null entries + override def createArray(stdType: StdType, size: Int): ArrayData[_] = SparkArrayData( null, stdType.underlyingType().asInstanceOf[ArrayType] ) - override def createMap(stdType: StdType): StdMap = SparkMap( + override def createMap(stdType: StdType): MapData[_, _] = SparkMapData( //TODO: make these as separate mutable standard spark types null, stdType.underlyingType().asInstanceOf[MapType] ) - override def createStruct(fieldTypes: JavaList[StdType]): StdStruct = { + override def createStruct(fieldTypes: JavaList[StdType]): RowData = { createStruct(null, fieldTypes) } - override def createStruct(fieldNames: JavaList[String], fieldTypes: JavaList[StdType]): StdStruct = { + override def createStruct(fieldNames: JavaList[String], fieldTypes: JavaList[StdType]): RowData = { val structFields = new Array[StructField](fieldTypes.size()) (0 until fieldTypes.size()).foreach({ idx => { @@ -59,13 +47,13 @@ class SparkFactory(private val _boundVariables: AbstractBoundVariables[DataType] ) } }) - SparkStruct(null, StructType(structFields)) + SparkRowData(null, StructType(structFields)) } - override def createStruct(stdType: StdType): StdStruct = { + override def createStruct(stdType: StdType): RowData = { //TODO: make these as separate mutable standard spark types val structType: StructType = stdType.underlyingType().asInstanceOf[StructType] - SparkStruct(null, structType) + SparkRowData(null, structType) } override def createStdType(typeSignature: String): StdType = SparkWrapper.createStdType( diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkWrapper.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkWrapper.scala index a52a0ca1..b365f716 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkWrapper.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/SparkWrapper.scala @@ -5,43 +5,71 @@ */ package com.linkedin.transport.spark -import com.linkedin.transport.api.data.StdData +import java.nio.ByteBuffer + +import com.linkedin.transport.api.data.PlatformData import com.linkedin.transport.api.types.StdType import com.linkedin.transport.spark.data._ import com.linkedin.transport.spark.types._ import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String object SparkWrapper { - def createStdData(data: Any, dataType: DataType): StdData = { // scalastyle:ignore cyclomatic.complexity + def createStdData(data: Any, dataType: DataType): Object = { // scalastyle:ignore cyclomatic.complexity if (data == null) { null } else { dataType match { - case _: IntegerType => SparkInteger(data.asInstanceOf[Integer]) - case _: LongType => SparkLong(data.asInstanceOf[java.lang.Long]) - case _: BooleanType => SparkBoolean(data.asInstanceOf[java.lang.Boolean]) - case _: StringType => SparkString(data.asInstanceOf[UTF8String]) - case _: ArrayType => SparkArray(data.asInstanceOf[ArrayData], dataType.asInstanceOf[ArrayType]) - case _: MapType => SparkMap(data.asInstanceOf[MapData], dataType.asInstanceOf[MapType]) - case _: StructType => SparkStruct(data.asInstanceOf[InternalRow], dataType.asInstanceOf[StructType]) + case _: IntegerType => data.asInstanceOf[Object] + case _: LongType => data.asInstanceOf[Object] + case _: BooleanType => data.asInstanceOf[Object] + case _: StringType => data.asInstanceOf[UTF8String].toString + case _: FloatType => data.asInstanceOf[Object] + case _: DoubleType => data.asInstanceOf[Object] + case _: BinaryType => ByteBuffer.wrap(data.asInstanceOf[Array[Byte]]) + case _: ArrayType => SparkArrayData( + data.asInstanceOf[org.apache.spark.sql.catalyst.util.ArrayData], dataType.asInstanceOf[ArrayType] + ) + case _: MapType => SparkMapData( + data.asInstanceOf[org.apache.spark.sql.catalyst.util.MapData], dataType.asInstanceOf[MapType] + ) + case _: StructType => SparkRowData(data.asInstanceOf[InternalRow], dataType.asInstanceOf[StructType]) case _: NullType => null case _ => throw new UnsupportedOperationException("Unrecognized Spark Type: " + dataType.getClass) } } } + def getPlatformData(transportData: Object): Object = { + if (transportData == null) { + null + } else { + transportData match { + case _: java.lang.Integer => transportData + case _: java.lang.Long => transportData + case _: java.lang.Float => transportData + case _: java.lang.Double => transportData + case _: java.lang.Boolean => transportData + case _: java.lang.String => UTF8String.fromString(transportData.asInstanceOf[String]) + case _: ByteBuffer => transportData.asInstanceOf[ByteBuffer].array() + case _ => transportData.asInstanceOf[PlatformData].getUnderlyingData + } + } + } + def createStdType(dataType: DataType): StdType = dataType match { case _: IntegerType => SparkIntegerType(dataType.asInstanceOf[IntegerType]) case _: LongType => SparkLongType(dataType.asInstanceOf[LongType]) case _: BooleanType => SparkBooleanType(dataType.asInstanceOf[BooleanType]) case _: StringType => SparkStringType(dataType.asInstanceOf[StringType]) + case _: FloatType => SparkFloatType(dataType.asInstanceOf[FloatType]) + case _: DoubleType => SparkDoubleType(dataType.asInstanceOf[DoubleType]) + case _: BinaryType => SparkBinaryType(dataType.asInstanceOf[BinaryType]) case _: ArrayType => SparkArrayType(dataType.asInstanceOf[ArrayType]) case _: MapType => SparkMapType(dataType.asInstanceOf[MapType]) - case _: StructType => SparkStructType(dataType.asInstanceOf[StructType]) + case _: StructType => SparkRowType(dataType.asInstanceOf[StructType]) case _: NullType => SparkUnknownType(dataType.asInstanceOf[NullType]) case _ => throw new UnsupportedOperationException("Unrecognized Spark Type: " + dataType.getClass) } diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/StdUdfWrapper.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/StdUdfWrapper.scala index c87f3bfc..5eca65a1 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/StdUdfWrapper.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/StdUdfWrapper.scala @@ -10,7 +10,6 @@ import java.nio.file.Paths import java.util.List import com.linkedin.transport.api.StdFactory -import com.linkedin.transport.api.data.{PlatformData, StdData} import com.linkedin.transport.api.udf._ import com.linkedin.transport.spark.typesystem.SparkTypeInference import com.linkedin.transport.utils.FileSystemUtils @@ -64,29 +63,29 @@ abstract class StdUdfWrapper(_expressions: Seq[Expression]) extends Expression if (wrappedConstants != null) { val requiredFiles = wrappedConstants.length match { case 0 => - _stdUdf.asInstanceOf[StdUDF0[StdData]].getRequiredFiles() + _stdUdf.asInstanceOf[StdUDF0[Object]].getRequiredFiles() case 1 => - _stdUdf.asInstanceOf[StdUDF1[StdData, StdData]].getRequiredFiles(wrappedConstants(0)) + _stdUdf.asInstanceOf[StdUDF1[Object, Object]].getRequiredFiles(wrappedConstants(0)) case 2 => - _stdUdf.asInstanceOf[StdUDF2[StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF2[Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1)) case 3 => - _stdUdf.asInstanceOf[StdUDF3[StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF3[Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2)) case 4 => - _stdUdf.asInstanceOf[StdUDF4[StdData, StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF4[Object, Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2), wrappedConstants(3)) case 5 => - _stdUdf.asInstanceOf[StdUDF5[StdData, StdData, StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF5[Object, Object, Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2), wrappedConstants(3), wrappedConstants(4)) case 6 => - _stdUdf.asInstanceOf[StdUDF6[StdData, StdData, StdData, StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF6[Object, Object, Object, Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2), wrappedConstants(3), wrappedConstants(4), wrappedConstants(5)) case 7 => - _stdUdf.asInstanceOf[StdUDF7[StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF7[Object, Object, Object, Object, Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2), wrappedConstants(3), wrappedConstants(4), wrappedConstants(5), wrappedConstants(6)) case 8 => - _stdUdf.asInstanceOf[StdUDF8[StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData]].getRequiredFiles(wrappedConstants(0), + _stdUdf.asInstanceOf[StdUDF8[Object, Object, Object, Object, Object, Object, Object, Object, Object]].getRequiredFiles(wrappedConstants(0), wrappedConstants(1), wrappedConstants(2), wrappedConstants(3), wrappedConstants(4), wrappedConstants(5), wrappedConstants(6), wrappedConstants(7)) case _ => throw new UnsupportedOperationException("getRequiredFiles not yet supported for StdUDF" + _expressions.length) @@ -95,7 +94,7 @@ abstract class StdUdfWrapper(_expressions: Seq[Expression]) extends Expression lazy val sparkContext = SparkSession.builder().getOrCreate().sparkContext _distributedCacheFiles = requiredFiles.map(file => { try { - val resolvedFile = FileSystemUtils.resolveLatest(file, FileSystemUtils.getHDFSFileSystem) + val resolvedFile = FileSystemUtils.resolveLatest(file) // TODO: Currently does not support adding of files with same file name. E.g dirA/file.txt dirB/file.txt sparkContext.addFile(resolvedFile) resolvedFile @@ -108,8 +107,8 @@ abstract class StdUdfWrapper(_expressions: Seq[Expression]) extends Expression } } // scalastyle:on magic.number - private final def checkNullsAndWrapConstants(): Array[StdData] = { - val wrappedConstants = new Array[StdData](_expressions.length) + private final def checkNullsAndWrapConstants(): Array[Object] = { + val wrappedConstants = new Array[Object](_expressions.length) for (i <- _expressions.indices) { val constantValue = if (_expressions(i).foldable) _expressions(i).eval() else null if (!_nullableArguments(i) && _expressions(i).foldable && constantValue == null) { @@ -135,42 +134,41 @@ abstract class StdUdfWrapper(_expressions: Seq[Expression]) extends Expression } val stdResult = wrappedArguments.length match { case 0 => - _stdUdf.asInstanceOf[StdUDF0[StdData]].eval() + _stdUdf.asInstanceOf[StdUDF0[Object]].eval() case 1 => - _stdUdf.asInstanceOf[StdUDF1[StdData, StdData]].eval(wrappedArguments(0)) + _stdUdf.asInstanceOf[StdUDF1[Object, Object]].eval(wrappedArguments(0)) case 2 => - _stdUdf.asInstanceOf[StdUDF2[StdData, StdData, StdData]].eval(wrappedArguments(0), wrappedArguments(1)) + _stdUdf.asInstanceOf[StdUDF2[Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1)) case 3 => - _stdUdf.asInstanceOf[StdUDF3[StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), wrappedArguments(1), + _stdUdf.asInstanceOf[StdUDF3[Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2)) case 4 => - _stdUdf.asInstanceOf[StdUDF4[StdData, StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), + _stdUdf.asInstanceOf[StdUDF4[Object, Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2), wrappedArguments(3)) case 5 => - _stdUdf.asInstanceOf[StdUDF5[StdData, StdData, StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), + _stdUdf.asInstanceOf[StdUDF5[Object, Object, Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2), wrappedArguments(3), wrappedArguments(4)) case 6 => - _stdUdf.asInstanceOf[StdUDF6[StdData, StdData, StdData, StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), + _stdUdf.asInstanceOf[StdUDF6[Object, Object, Object, Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2), wrappedArguments(3), wrappedArguments(4), wrappedArguments(5)) case 7 => - _stdUdf.asInstanceOf[StdUDF7[StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), + _stdUdf.asInstanceOf[StdUDF7[Object, Object, Object, Object, Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2), wrappedArguments(3), wrappedArguments(4), wrappedArguments(5), wrappedArguments(6)) case 8 => - _stdUdf.asInstanceOf[StdUDF8[StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData, StdData]].eval(wrappedArguments(0), + _stdUdf.asInstanceOf[StdUDF8[Object, Object, Object, Object, Object, Object, Object, Object, Object]].eval(wrappedArguments(0), wrappedArguments(1), wrappedArguments(2), wrappedArguments(3), wrappedArguments(4), wrappedArguments(5), wrappedArguments(6), wrappedArguments(7)) case _ => throw new UnsupportedOperationException("eval not yet supported for StdUDF" + _expressions.length) } - if (stdResult == null) null else stdResult.asInstanceOf[PlatformData].getUnderlyingData + SparkWrapper.getPlatformData(stdResult) } } // scalastyle:on magic.number - - private final def checkNullsAndWrapArguments(input: InternalRow): Array[StdData] = { - val wrappedArguments = new Array[StdData](_expressions.length) + private final def checkNullsAndWrapArguments(input: InternalRow): Array[Object] = { + val wrappedArguments = new Array[Object](_expressions.length) for (i <- _expressions.indices) { val evaluatedExpression = _expressions(i).eval(input) if(!_nullableArguments(i) && evaluatedExpression == null) { diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArray.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArrayData.scala similarity index 75% rename from transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArray.scala rename to transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArrayData.scala index 9fe91cab..e98ef069 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArray.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkArrayData.scala @@ -7,20 +7,19 @@ package com.linkedin.transport.spark.data import java.util -import com.linkedin.transport.api.data.{PlatformData, StdArray, StdData} +import com.linkedin.transport.api.data.{ArrayData, PlatformData} import com.linkedin.transport.spark.SparkWrapper -import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{ArrayType, DataType} import scala.collection.mutable.ArrayBuffer -case class SparkArray(private var _arrayData: ArrayData, - private val _arrayType: DataType) extends StdArray with PlatformData { +case class SparkArrayData[E](private var _arrayData: org.apache.spark.sql.catalyst.util.ArrayData, + private val _arrayType: DataType) extends ArrayData[E] with PlatformData { private val _elementType = _arrayType.asInstanceOf[ArrayType].elementType private var _mutableBuffer: ArrayBuffer[Any] = if (_arrayData == null) createMutableArray() else null - override def add(e: StdData): Unit = { + override def add(e: E): Unit = { // Once add is called, we cannot use Spark's readonly ArrayData API // we have to add elements to a mutable buffer and start using that // always instead of the readonly stdType @@ -29,7 +28,7 @@ case class SparkArray(private var _arrayData: ArrayData, _mutableBuffer = createMutableArray() } // TODO: Does not support inserting nulls. Should we? - _mutableBuffer.append(e.asInstanceOf[PlatformData].getUnderlyingData) + _mutableBuffer.append(SparkWrapper.getPlatformData(e.asInstanceOf[Object])) } private def createMutableArray(): ArrayBuffer[Any] = { @@ -47,20 +46,20 @@ case class SparkArray(private var _arrayData: ArrayData, if (_mutableBuffer == null) { _arrayData } else { - ArrayData.toArrayData(_mutableBuffer) + org.apache.spark.sql.catalyst.util.ArrayData.toArrayData(_mutableBuffer) } } override def setUnderlyingData(value: scala.Any): Unit = { - _arrayData = value.asInstanceOf[ArrayData] + _arrayData = value.asInstanceOf[org.apache.spark.sql.catalyst.util.ArrayData] _mutableBuffer = null } - override def iterator(): util.Iterator[StdData] = { - new util.Iterator[StdData] { + override def iterator(): util.Iterator[E] = { + new util.Iterator[E] { private var idx = 0 - override def next(): StdData = { + override def next(): E = { val e = get(idx) idx += 1 e @@ -78,11 +77,11 @@ case class SparkArray(private var _arrayData: ArrayData, } } - override def get(idx: Int): StdData = { + override def get(idx: Int): E = { if (_mutableBuffer == null) { - SparkWrapper.createStdData(_arrayData.get(idx, _elementType), _elementType) + SparkWrapper.createStdData(_arrayData.get(idx, _elementType), _elementType).asInstanceOf[E] } else { - SparkWrapper.createStdData(_mutableBuffer(idx), _elementType) + SparkWrapper.createStdData(_mutableBuffer(idx), _elementType).asInstanceOf[E] } } } diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkBoolean.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkBoolean.scala deleted file mode 100644 index 2477eef2..00000000 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkBoolean.scala +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.spark.data - -import com.linkedin.transport.api.data.{PlatformData, StdBoolean} - -case class SparkBoolean(private var _bool: java.lang.Boolean) extends StdBoolean with PlatformData { - - override def get(): Boolean = _bool.booleanValue() - - override def getUnderlyingData: AnyRef = _bool - - override def setUnderlyingData(value: scala.Any): Unit = _bool = value.asInstanceOf[java.lang.Boolean] -} diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkInteger.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkInteger.scala deleted file mode 100644 index b7c0db9e..00000000 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkInteger.scala +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.spark.data - -import com.linkedin.transport.api.data.{PlatformData, StdInteger} - -case class SparkInteger(private var _int: Integer) extends StdInteger with PlatformData { - - override def get(): Int = _int.intValue() - - override def getUnderlyingData: AnyRef = _int - - override def setUnderlyingData(value: scala.Any): Unit = _int = value.asInstanceOf[Integer] -} diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkLong.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkLong.scala deleted file mode 100644 index 5a534290..00000000 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkLong.scala +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.spark.data - -import com.linkedin.transport.api.data.{PlatformData, StdLong} - -case class SparkLong(private var _long: java.lang.Long) extends StdLong with PlatformData { - - override def get(): Long = _long.longValue() - - override def getUnderlyingData: AnyRef = _long - - override def setUnderlyingData(value: scala.Any): Unit = _long = value.asInstanceOf[java.lang.Long] - -} diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMap.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMapData.scala similarity index 57% rename from transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMap.scala rename to transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMapData.scala index 4859d92e..cd9679c8 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMap.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkMapData.scala @@ -7,41 +7,44 @@ package com.linkedin.transport.spark.data import java.util -import com.linkedin.transport.api.data.{PlatformData, StdData, StdMap} +import com.linkedin.transport.api.data.{MapData, PlatformData} import com.linkedin.transport.spark.SparkWrapper -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, MapData} +import org.apache.spark.sql.catalyst.util.ArrayBasedMapData import org.apache.spark.sql.types.MapType import scala.collection.mutable.Map -case class SparkMap(private var _mapData: MapData, - private val _mapType: MapType) extends StdMap with PlatformData { +case class SparkMapData[K, V](private var _mapData: org.apache.spark.sql.catalyst.util.MapData, + private val _mapType: MapType) extends MapData[K, V] with PlatformData { private val _keyType = _mapType.keyType private val _valueType = _mapType.valueType private var _mutableMap: Map[Any, Any] = if (_mapData == null) createMutableMap() else null - override def put(key: StdData, value: StdData): Unit = { + override def put(key: K, value: V): Unit = { // TODO: Does not support inserting nulls. Should we? if (_mutableMap == null) { _mutableMap = createMutableMap() } - _mutableMap.put(key.asInstanceOf[PlatformData].getUnderlyingData, value.asInstanceOf[PlatformData].getUnderlyingData) + _mutableMap.put( + SparkWrapper.getPlatformData(key.asInstanceOf[Object]), + SparkWrapper.getPlatformData(value.asInstanceOf[Object]) + ) } - override def keySet(): util.Set[StdData] = { - new util.AbstractSet[StdData] { + override def keySet(): util.Set[K] = { + new util.AbstractSet[K] { - override def iterator(): util.Iterator[StdData] = new util.Iterator[StdData] { + override def iterator(): util.Iterator[K] = new util.Iterator[K] { private val keysIterator = if (_mutableMap == null) _mapData.keyArray().array.iterator else _mutableMap.keysIterator - override def next(): StdData = SparkWrapper.createStdData(keysIterator.next(), _keyType) + override def next(): K = SparkWrapper.createStdData(keysIterator.next(), _keyType).asInstanceOf[K] override def hasNext: Boolean = keysIterator.hasNext } - override def size(): Int = SparkMap.this.size() + override def size(): Int = SparkMapData.this.size() } } @@ -53,30 +56,31 @@ case class SparkMap(private var _mapData: MapData, } } - override def values(): util.Collection[StdData] = { - new util.AbstractCollection[StdData] { + override def values(): util.Collection[V] = { + new util.AbstractCollection[V] { - override def iterator(): util.Iterator[StdData] = new util.Iterator[StdData] { + override def iterator(): util.Iterator[V] = new util.Iterator[V] { private val valueIterator = if (_mutableMap == null) _mapData.valueArray().array.iterator else _mutableMap.valuesIterator - override def next(): StdData = SparkWrapper.createStdData(valueIterator.next(), _valueType) + override def next(): V = SparkWrapper.createStdData(valueIterator.next(), _valueType).asInstanceOf[V] override def hasNext: Boolean = valueIterator.hasNext } - override def size(): Int = SparkMap.this.size() + override def size(): Int = SparkMapData.this.size() } } - override def containsKey(key: StdData): Boolean = get(key) != null + override def containsKey(key: K): Boolean = get(key) != null - override def get(key: StdData): StdData = { + override def get(key: K): V = { // Spark's complex data types (MapData, ArrayData, InternalRow) do not implement equals/hashcode // If the key is of the above complex data types, get() will return null if (_mutableMap == null) { _mutableMap = createMutableMap() } - SparkWrapper.createStdData(_mutableMap.get(key.asInstanceOf[PlatformData].getUnderlyingData).orNull, _valueType) + SparkWrapper.createStdData(_mutableMap.get(SparkWrapper.getPlatformData(key.asInstanceOf[Object])).orNull, _valueType) + .asInstanceOf[V] } private def createMutableMap(): Map[Any, Any] = { @@ -96,7 +100,7 @@ case class SparkMap(private var _mapData: MapData, } override def setUnderlyingData(value: scala.Any): Unit = { - _mapData = value.asInstanceOf[MapData] + _mapData = value.asInstanceOf[org.apache.spark.sql.catalyst.util.MapData] _mutableMap = null } } diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkStruct.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkRowData.scala similarity index 74% rename from transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkStruct.scala rename to transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkRowData.scala index ba432905..9cbc883e 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkStruct.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkRowData.scala @@ -7,7 +7,7 @@ package com.linkedin.transport.spark.data import java.util.{List => JavaList} -import com.linkedin.transport.api.data.{PlatformData, StdData, StdStruct} +import com.linkedin.transport.api.data.{PlatformData, RowData} import com.linkedin.transport.spark.SparkWrapper import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.StructType @@ -16,14 +16,14 @@ import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -case class SparkStruct(private var _row: InternalRow, - private val _structType: StructType) extends StdStruct with PlatformData { +case class SparkRowData(private var _row: InternalRow, + private val _structType: StructType) extends RowData with PlatformData { private var _mutableBuffer: ArrayBuffer[Any] = if (_row == null) createMutableStruct() else null - override def getField(name: String): StdData = getField(_structType.fieldIndex(name)) + override def getField(name: String): Object = getField(_structType.fieldIndex(name)) - override def getField(index: Int): StdData = { + override def getField(index: Int): Object = { val fieldDataType = _structType(index).dataType if (_mutableBuffer == null) { SparkWrapper.createStdData(_row.get(index, fieldDataType), fieldDataType) @@ -32,15 +32,15 @@ case class SparkStruct(private var _row: InternalRow, } } - override def setField(name: String, value: StdData): Unit = { + override def setField(name: String, value: Object): Unit = { setField(_structType.fieldIndex(name), value) } - override def setField(index: Int, value: StdData): Unit = { + override def setField(index: Int, value: Object): Unit = { if (_mutableBuffer == null) { _mutableBuffer = createMutableStruct() } - _mutableBuffer(index) = value.asInstanceOf[PlatformData].getUnderlyingData + _mutableBuffer(index) = SparkWrapper.getPlatformData(value) } private def createMutableStruct() = { @@ -51,7 +51,7 @@ case class SparkStruct(private var _row: InternalRow, } } - override def fields(): JavaList[StdData] = { + override def fields(): JavaList[Object] = { _structType.indices.map(getField).asJava } diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkString.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkString.scala deleted file mode 100644 index bd089dd5..00000000 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/data/SparkString.scala +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.spark.data - -import com.linkedin.transport.api.data.{PlatformData, StdString} -import org.apache.spark.unsafe.types.UTF8String - -case class SparkString(private var _str: UTF8String) extends StdString with PlatformData { - - override def get(): String = _str.toString - - override def getUnderlyingData: AnyRef = _str - - override def setUnderlyingData(value: scala.Any): Unit = _str = value.asInstanceOf[UTF8String] -} diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/types/SparkTypes.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/types/SparkTypes.scala index b9199565..554a282d 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/types/SparkTypes.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/types/SparkTypes.scala @@ -29,6 +29,21 @@ case class SparkStringType(stringType: StringType) extends StdStringType { override def underlyingType(): DataType = stringType } +case class SparkFloatType(floatType: FloatType) extends StdFloatType { + + override def underlyingType(): DataType = floatType +} + +case class SparkDoubleType(doubleType: DoubleType) extends StdDoubleType { + + override def underlyingType(): DataType = doubleType +} + +case class SparkBinaryType(bytesType: BinaryType) extends StdBinaryType { + + override def underlyingType(): DataType = bytesType +} + case class SparkBooleanType(booleanType: BooleanType) extends StdBooleanType { override def underlyingType(): DataType = booleanType @@ -55,7 +70,7 @@ case class SparkMapType(mapType: MapType) extends StdMapType { override def valueType(): StdType = SparkWrapper.createStdType(mapType.valueType) } -case class SparkStructType(structType: StructType) extends StdStructType { +case class SparkRowType(structType: StructType) extends RowType { override def underlyingType(): DataType = structType diff --git a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/typesystem/SparkTypeSystem.scala b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/typesystem/SparkTypeSystem.scala index 81dcc526..a7c66fe7 100644 --- a/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/typesystem/SparkTypeSystem.scala +++ b/transportable-udfs-spark/src/main/scala/com/linkedin/transport/spark/typesystem/SparkTypeSystem.scala @@ -33,6 +33,12 @@ class SparkTypeSystem extends AbstractTypeSystem[DataType] { override protected def createStringType(): DataType = StringType + override protected def createFloatType(): DataType = FloatType + + override protected def createDoubleType(): DataType = DoubleType + + override protected def createBinaryType(): DataType = BinaryType + override protected def createUnknownType(): DataType = NullType override protected def createArrayType(elementType: DataType): DataType = @@ -65,4 +71,10 @@ class SparkTypeSystem extends AbstractTypeSystem[DataType] { override protected def isMapType(dataType: DataType): Boolean = dataType.isInstanceOf[MapType] override protected def isStructType(dataType: DataType): Boolean = dataType.isInstanceOf[StructType] + + override protected def isFloatType(dataType: DataType): Boolean = dataType.isInstanceOf[FloatType] + + override protected def isDoubleType(dataType: DataType): Boolean = dataType.isInstanceOf[DoubleType] + + override protected def isBinaryType(dataType: DataType): Boolean = dataType.isInstanceOf[BinaryType] } diff --git a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/TestSparkFactory.scala b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/TestSparkFactory.scala index 3b928045..20221e47 100644 --- a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/TestSparkFactory.scala +++ b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/TestSparkFactory.scala @@ -5,6 +5,9 @@ */ package com.linkedin.transport.spark +import java.nio.ByteBuffer +import java.nio.charset.Charset + import com.linkedin.transport.api.data.PlatformData import com.linkedin.transport.spark.typesystem.{SparkBoundVariables, SparkTypeFactory} import org.apache.spark.sql.catalyst.InternalRow @@ -20,14 +23,6 @@ class TestSparkFactory { val typeFactory: SparkTypeFactory = new SparkTypeFactory val stdFactory = new SparkFactory(new SparkBoundVariables) - @Test - def testCreatePrimitives(): Unit = { - assertEquals(stdFactory.createInteger(1).get(), 1) - assertEquals(stdFactory.createLong(1L).get(), 1L) - assertEquals(stdFactory.createBoolean(true).get(), true) - assertEquals(stdFactory.createString("").get(), "") - } - @Test def testCreateArray(): Unit = { var stdArray = stdFactory.createArray(stdFactory.createStdType("array(integer)")) @@ -54,38 +49,40 @@ class TestSparkFactory { @Test def testCreateStructFromStdType(): Unit = { - val fieldNames = Array("strField", "intField", "longField", "boolField", "arrField") - val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "array(integer)") + val fieldNames = Array("strField", "intField", "longField", "boolField", "floatField", "doubleField", + "bytesField", "arrField") + val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "real", "double", "varbinary", "array(integer)") val stdStruct = stdFactory.createStruct(stdFactory.createStdType(fieldNames.zip(fieldTypes).map(x => x._1 + " " + x._2).mkString("row(", ", ", ")"))) val internalRow = stdStruct.asInstanceOf[PlatformData].getUnderlyingData.asInstanceOf[InternalRow] assertEquals(internalRow.numFields, fieldTypes.length) - (0 until 5).foreach(idx => { + (0 until 8).foreach(idx => { assertEquals(internalRow.get(idx, stdFactory.createStdType(fieldTypes(idx)).underlyingType().asInstanceOf[DataType]), null) }) } @Test def testCreateStructFromFieldNamesAndTypes(): Unit = { - val fieldNames = Array("strField", "intField", "longField", "boolField", "arrField") - val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "array(integer)") + val fieldNames = Array("strField", "intField", "longField", "boolField", "floatField", "doubleField", + "bytesField", "arrField") + val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "real", "double", "varbinary", "array(integer)") val stdStruct = stdFactory.createStruct(fieldNames.toList.asJava, fieldTypes.map(stdFactory.createStdType).toList.asJava) val internalRow = stdStruct.asInstanceOf[PlatformData].getUnderlyingData.asInstanceOf[InternalRow] assertEquals(internalRow.numFields, fieldTypes.length) - (0 until 5).foreach(idx => { + (0 until 8).foreach(idx => { assertEquals(internalRow.get(idx, stdFactory.createStdType(fieldTypes(idx)).underlyingType().asInstanceOf[DataType]), null) }) } @Test def testCreateStructFromFieldTypes(): Unit = { - val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "array(integer)") + val fieldTypes = Array("varchar", "integer", "bigint", "boolean", "real", "double", "varbinary ", "array(integer)") val stdStruct = stdFactory.createStruct(fieldTypes.map(stdFactory.createStdType).toList.asJava) val internalRow = stdStruct.asInstanceOf[PlatformData].getUnderlyingData.asInstanceOf[InternalRow] assertEquals(internalRow.numFields, fieldTypes.length) - (0 until 5).foreach(idx => { + (0 until 8).foreach(idx => { assertEquals(internalRow.get(idx, stdFactory.createStdType(fieldTypes(idx)).underlyingType().asInstanceOf[DataType]), null) }) } diff --git a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkArray.scala b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkArray.scala index 00d70d88..dfc024ac 100644 --- a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkArray.scala +++ b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkArray.scala @@ -5,7 +5,8 @@ */ package com.linkedin.transport.spark.data -import com.linkedin.transport.api.data.{PlatformData, StdArray} +import com.linkedin.transport.api.data +import com.linkedin.transport.api.data.{PlatformData} import com.linkedin.transport.spark.{SparkFactory, SparkWrapper} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{ArrayType, DataTypes} @@ -20,35 +21,33 @@ class TestSparkArray { @Test def testCreateSparkArray(): Unit = { - val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[StdArray] + val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[data.ArrayData[Integer]] assertEquals(stdArray.size(), arrayData.numElements()) assertSame(stdArray.asInstanceOf[PlatformData].getUnderlyingData, arrayData) } @Test def testSparkArrayGet(): Unit = { - val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[StdArray] + val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[data.ArrayData[Integer]] (0 until stdArray.size).foreach(idx => { - assertEquals(stdArray.get(idx).asInstanceOf[SparkInteger].get(), idx) + assertEquals(stdArray.get(idx), idx) }) } @Test def testSparkArrayAdd(): Unit = { - val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[StdArray] - val insert = stdFactory.createInteger(5) // scalastyle:ignore magic.number - stdArray.add(insert) + val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[data.ArrayData[Integer]] + stdArray.add(5) // Since original ArrayData is immutable, a mutable ArrayBuffer should be created and set as the underlying object assertNotSame(stdArray.asInstanceOf[PlatformData].getUnderlyingData, arrayData) assertEquals(stdArray.size(), arrayData.numElements() + 1) - assertEquals(stdArray.get(stdArray.size() - 1), insert) + assertEquals(stdArray.get(stdArray.size() - 1), 5) } @Test def testSparkArrayMutabilityReset(): Unit = { - val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[StdArray] - val insert = stdFactory.createInteger(5) // scalastyle:ignore magic.number - stdArray.add(insert) + val stdArray = SparkWrapper.createStdData(arrayData, arrayType).asInstanceOf[data.ArrayData[Integer]] + stdArray.add(5) stdArray.asInstanceOf[PlatformData].setUnderlyingData(arrayData) // After underlying data is explicitly set, mutuable buffer should be removed assertSame(stdArray.asInstanceOf[PlatformData].getUnderlyingData, arrayData) diff --git a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkMap.scala b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkMap.scala index 608c027f..12675eb1 100644 --- a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkMap.scala +++ b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkMap.scala @@ -5,7 +5,7 @@ */ package com.linkedin.transport.spark.data -import com.linkedin.transport.api.data.{PlatformData, StdMap, StdString} +import com.linkedin.transport.api.data.{MapData, PlatformData} import com.linkedin.transport.spark.{SparkFactory, SparkWrapper} import org.apache.spark.sql.catalyst.util.ArrayBasedMapData import org.apache.spark.sql.types.{DataTypes, MapType} @@ -23,58 +23,54 @@ class TestSparkMap { @Test def testCreateSparkMap(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] assertEquals(stdMap.size(), mapData.numElements()) assertSame(stdMap.asInstanceOf[PlatformData].getUnderlyingData, mapData) } @Test def testSparkMapKeySet(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] - assertEqualsNoOrder(stdMap.keySet().toArray, mapData.keyArray.array.map(s => stdFactory.createString(s.toString))) + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] + assertEqualsNoOrder(stdMap.keySet().toArray, mapData.keyArray.array.map(s => s.toString)) } @Test def testSparkMapValues(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] - assertEqualsNoOrder(stdMap.values().toArray, mapData.valueArray.array.map(s => stdFactory.createString(s.toString))) + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] + assertEqualsNoOrder(stdMap.values().toArray, mapData.valueArray.array.map(s => s.toString)) } @Test def testSparkMapGet(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] mapData.keyArray.foreach(mapType.keyType, (idx, key) => { - assertEquals(stdMap.get(stdFactory.createString(key.toString)).asInstanceOf[StdString].get, + assertEquals(stdMap.get(key.toString), mapData.valueArray.array(idx).toString) }) - assertEquals(stdMap.containsKey(stdFactory.createString("nonExistentKey")), false) - // Even for a get in SparkMap we create mutable Map since Spark's Impl is based of arrays. So underlying object should change + assertEquals(stdMap.containsKey("nonExistentKey"), false) + // Even for a get in SparkMapData we create mutable Map since Spark's Impl is based of arrays. So underlying object should change assertNotSame(stdMap.asInstanceOf[PlatformData].getUnderlyingData, mapData) } @Test def testSparkMapContainsKey(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] - assertEquals(stdMap.containsKey(stdFactory.createString("k3")), true) - assertEquals(stdMap.containsKey(stdFactory.createString("k4")), false) + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] + assertEquals(stdMap.containsKey("k3"), true) + assertEquals(stdMap.containsKey("k4"), false) } @Test def testSparkMapPut(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] - val insertKey = stdFactory.createString("k4") - val insertVal = stdFactory.createString("v4") - stdMap.put(insertKey, insertVal) + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] + stdMap.put("k4", "v4") assertEquals(stdMap.size(), mapData.numElements() + 1) - assertEquals(stdMap.get(stdFactory.createString("k4")), insertVal) + assertEquals(stdMap.get("k4"), "v4") } @Test def testSparkMapMutabilityReset(): Unit = { - val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[StdMap] - val insertKey = stdFactory.createString("k4") - val insertVal = stdFactory.createString("v4") - stdMap.put(insertKey, insertVal) + val stdMap = SparkWrapper.createStdData(mapData, mapType).asInstanceOf[MapData[String, String]] + stdMap.put("k4", "v4") stdMap.asInstanceOf[PlatformData].setUnderlyingData(mapData) // After underlying data is explicitly set, mutuable map should be removed assertSame(stdMap.asInstanceOf[PlatformData].getUnderlyingData, mapData) diff --git a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkPrimitives.scala b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkPrimitives.scala deleted file mode 100644 index 34834732..00000000 --- a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkPrimitives.scala +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.spark.data - -import java.lang - -import com.linkedin.transport.api.data._ -import com.linkedin.transport.spark.{SparkFactory, SparkWrapper} -import org.apache.spark.sql.types.DataTypes -import org.apache.spark.unsafe.types.UTF8String -import org.testng.Assert.{assertEquals, assertSame} -import org.testng.annotations.Test - - -class TestSparkPrimitives { - - val stdFactory = new SparkFactory(null) - - @Test - def testCreateSparkInteger(): Unit = { - val intData = 123 - val stdInteger = SparkWrapper.createStdData(intData, DataTypes.IntegerType).asInstanceOf[StdInteger] - assertEquals(stdInteger.get(), intData) - assertSame(stdInteger.asInstanceOf[PlatformData].getUnderlyingData, intData) - } - - @Test - def testCreateSparkLong(): Unit = { - val longData = new lang.Long(1234L) // scalastyle:ignore magic.number - val stdLong = SparkWrapper.createStdData(longData, DataTypes.LongType).asInstanceOf[StdLong] - assertEquals(stdLong.get(), longData) - assertSame(stdLong.asInstanceOf[PlatformData].getUnderlyingData, longData) - } - - @Test - def testCreateSparkBoolean(): Unit = { - val booleanData = new lang.Boolean(true) - val stdBoolean = SparkWrapper.createStdData(booleanData, DataTypes.BooleanType).asInstanceOf[StdBoolean] - assertEquals(stdBoolean.get(), true) - assertSame(stdBoolean.asInstanceOf[PlatformData].getUnderlyingData, booleanData) - } - - @Test - def testCreateSparkString(): Unit = { - val stringData = UTF8String.fromString("test") - val stdString = SparkWrapper.createStdData(stringData, DataTypes.StringType).asInstanceOf[StdString] - assertEquals(stdString.get(), "test") - assertSame(stdString.asInstanceOf[PlatformData].getUnderlyingData, stringData) - } - -} diff --git a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkStruct.scala b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkRowData.scala similarity index 70% rename from transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkStruct.scala rename to transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkRowData.scala index 9a911af2..df7def17 100644 --- a/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkStruct.scala +++ b/transportable-udfs-spark/src/test/scala/com/linkedin/transport/spark/data/TestSparkRowData.scala @@ -5,7 +5,7 @@ */ package com.linkedin.transport.spark.data -import com.linkedin.transport.api.data.{PlatformData, StdStruct} +import com.linkedin.transport.api.data.{PlatformData, RowData} import com.linkedin.transport.spark.{SparkFactory, SparkWrapper} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.util.ArrayData @@ -14,7 +14,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.testng.Assert.{assertEquals, assertNotSame, assertSame} import org.testng.annotations.Test -class TestSparkStruct { +class TestSparkRowData { val stdFactory = new SparkFactory(null) val dataArray = Array(UTF8String.fromString("str1"), 0, 2L, false, ArrayData.toArrayData(Array.range(0, 5))) // scalastyle:ignore magic.number val fieldNames = Array("strField", "intField", "longField", "boolField", "arrField") @@ -25,41 +25,41 @@ class TestSparkStruct { @Test def testCreateSparkStruct(): Unit = { - val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[StdStruct] + val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[RowData] assertSame(stdStruct.asInstanceOf[PlatformData].getUnderlyingData, structData) } @Test def testSparkStructGetField(): Unit = { - val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[StdStruct] + val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[RowData] dataArray.indices.foreach(idx => { - assertEquals(stdStruct.getField(idx).asInstanceOf[PlatformData].getUnderlyingData, dataArray(idx)) - assertEquals(stdStruct.getField(fieldNames(idx)).asInstanceOf[PlatformData].getUnderlyingData, dataArray(idx)) + assertEquals(SparkWrapper.getPlatformData(stdStruct.getField(idx)), dataArray(idx)) + assertEquals(SparkWrapper.getPlatformData(stdStruct.getField(fieldNames(idx))), dataArray(idx)) }) } @Test def testSparkStructFields(): Unit = { - val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[StdStruct] + val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[RowData] assertEquals(stdStruct.fields().size(), structData.numFields) - assertEquals(stdStruct.fields().toArray.map(f => f.asInstanceOf[PlatformData].getUnderlyingData), dataArray) + assertEquals(stdStruct.fields().toArray.map(f => SparkWrapper.getPlatformData(f)), dataArray) } @Test def testSparkStructSetField(): Unit = { - val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[StdStruct] - stdStruct.setField(1, stdFactory.createInteger(1)) - assertEquals(stdStruct.getField(1).asInstanceOf[PlatformData].getUnderlyingData, 1) - stdStruct.setField(fieldNames(2), stdFactory.createLong(5)) // scalastyle:ignore magic.number - assertEquals(stdStruct.getField(fieldNames(2)).asInstanceOf[PlatformData].getUnderlyingData, 5L) // scalastyle:ignore magic.number + val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[RowData] + stdStruct.setField(1, 1) + assertEquals(stdStruct.getField(1), 1) + stdStruct.setField(fieldNames(2), 5L) // scalastyle:ignore magic.number + assertEquals(stdStruct.getField(fieldNames(2)), 5L) // scalastyle:ignore magic.number // Since original InternalRow is immutable, a mutable ArrayBuffer should be created and set as the underlying object assertNotSame(stdStruct.asInstanceOf[PlatformData].getUnderlyingData, structData) } @Test def testSparkStructMutabilityReset(): Unit = { - val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[StdStruct] - stdStruct.setField(1, stdFactory.createInteger(1)) + val stdStruct = SparkWrapper.createStdData(structData, structType).asInstanceOf[RowData] + stdStruct.setField(1, 1) stdStruct.asInstanceOf[PlatformData].setUnderlyingData(structData) // After underlying data is explicitly set, mutable buffer should be removed assertSame(stdStruct.asInstanceOf[PlatformData].getUnderlyingData, structData) diff --git a/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java b/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java index 7dbc3d34..4e85393e 100644 --- a/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java +++ b/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java @@ -6,7 +6,9 @@ package com.linkedin.transport.test; import com.google.common.base.Preconditions; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.TopLevelStdUDF; import com.linkedin.transport.test.spi.FunctionCall; @@ -26,15 +28,12 @@ * An abstract class to be extended by all test classes. This class contains helper methods to initialize the * {@link StdTester} and create input and output data for the test cases. * - * The mapping between a {@link StdData} to the corresponding Java type is given below: + * Primitive data is represented by primitive types when passed to the test cases. + * The mapping between container types to the corresponding Java type is given below: *
    - *
  • {@link com.linkedin.transport.api.data.StdInteger} = {@link Integer}
    • - *
  • {@link com.linkedin.transport.api.data.StdLong} = {@link Long}
    • - *
  • {@link com.linkedin.transport.api.data.StdBoolean} = {@link Boolean}
    • - *
  • {@link com.linkedin.transport.api.data.StdString} = {@link String}
    • - *
  • {@link com.linkedin.transport.api.data.StdArray} = Use {@link #array(Object...)} to create arrays
    • - *
  • {@link com.linkedin.transport.api.data.StdMap} = Use {@link #map(Object...)} to create maps
    • - *
  • {@link com.linkedin.transport.api.data.StdStruct} = Use {@link #row(Object...)} to create structs
    • + *
  • {@link ArrayData} = Use {@link #array(Object...)} to create arrays
    • + *
  • {@link MapData} = Use {@link #map(Object...)} to create maps
    • + *
  • {@link RowData} = Use {@link #row(Object...)} to create structs
    • *
* * diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericFactory.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericFactory.java index a84417bb..d3d26338 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericFactory.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericFactory.java @@ -5,22 +5,13 @@ */ package com.linkedin.transport.test.generic; -import com.google.common.base.Preconditions; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdBoolean; -import com.linkedin.transport.api.data.StdInteger; -import com.linkedin.transport.api.data.StdLong; -import com.linkedin.transport.api.data.StdMap; -import com.linkedin.transport.api.data.StdString; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.test.generic.data.GenericArray; -import com.linkedin.transport.test.generic.data.GenericBoolean; -import com.linkedin.transport.test.generic.data.GenericInteger; -import com.linkedin.transport.test.generic.data.GenericLong; -import com.linkedin.transport.test.generic.data.GenericMap; -import com.linkedin.transport.test.generic.data.GenericString; +import com.linkedin.transport.test.generic.data.GenericArrayData; +import com.linkedin.transport.test.generic.data.GenericMapData; import com.linkedin.transport.test.generic.data.GenericStruct; import com.linkedin.transport.test.generic.typesystem.GenericTypeFactory; import com.linkedin.transport.test.spi.types.TestType; @@ -43,54 +34,33 @@ public GenericFactory(AbstractBoundVariables boundVariables) { } @Override - public StdInteger createInteger(int value) { - return new GenericInteger(value); + public ArrayData createArray(StdType stdType, int expectedSize) { + return new GenericArrayData(new ArrayList<>(expectedSize), (TestType) stdType.underlyingType()); } @Override - public StdLong createLong(long value) { - return new GenericLong(value); - } - - @Override - public StdBoolean createBoolean(boolean value) { - return new GenericBoolean(value); - } - - @Override - public StdString createString(String value) { - Preconditions.checkNotNull(value, "Cannot create a null StdString"); - return new GenericString(value); - } - - @Override - public StdArray createArray(StdType stdType, int expectedSize) { - return new GenericArray(new ArrayList<>(expectedSize), (TestType) stdType.underlyingType()); - } - - @Override - public StdArray createArray(StdType stdType) { + public ArrayData createArray(StdType stdType) { return createArray(stdType, 0); } @Override - public StdMap createMap(StdType stdType) { - return new GenericMap((TestType) stdType.underlyingType()); + public MapData createMap(StdType stdType) { + return new GenericMapData((TestType) stdType.underlyingType()); } @Override - public StdStruct createStruct(List fieldNames, List fieldTypes) { + public RowData createStruct(List fieldNames, List fieldTypes) { return new GenericStruct(TestTypeFactory.struct(fieldNames, fieldTypes.stream().map(x -> (TestType) x.underlyingType()).collect(Collectors.toList()))); } @Override - public StdStruct createStruct(List fieldTypes) { + public RowData createStruct(List fieldTypes) { return createStruct(null, fieldTypes); } @Override - public StdStruct createStruct(StdType stdType) { + public RowData createStruct(StdType stdType) { return new GenericStruct((TestType) stdType.underlyingType()); } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericQueryExecutor.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericQueryExecutor.java index 09a83004..0c4d17dd 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericQueryExecutor.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericQueryExecutor.java @@ -9,6 +9,9 @@ import com.linkedin.transport.test.spi.Row; import com.linkedin.transport.test.spi.types.ArrayTestType; import com.linkedin.transport.test.spi.types.BooleanTestType; +import com.linkedin.transport.test.spi.types.BinaryTestType; +import com.linkedin.transport.test.spi.types.DoubleTestType; +import com.linkedin.transport.test.spi.types.FloatTestType; import com.linkedin.transport.test.spi.types.IntegerTestType; import com.linkedin.transport.test.spi.types.LongTestType; import com.linkedin.transport.test.spi.types.MapTestType; @@ -62,9 +65,15 @@ private Pair resolveFunctionCall(FunctionCall call) { private Pair resolveParameter(Object argument, TestType argumentType) { if (argument instanceof FunctionCall) { return resolveFunctionCall((FunctionCall) argument); - } else if (argument == null || argumentType instanceof UnknownTestType || argumentType instanceof IntegerTestType - || argumentType instanceof LongTestType || argumentType instanceof BooleanTestType - || argumentType instanceof StringTestType) { + } else if (argument == null + || argumentType instanceof UnknownTestType + || argumentType instanceof IntegerTestType + || argumentType instanceof LongTestType + || argumentType instanceof BooleanTestType + || argumentType instanceof StringTestType + || argumentType instanceof FloatTestType + || argumentType instanceof DoubleTestType + || argumentType instanceof BinaryTestType) { return Pair.of(argumentType, argument); } else if (argumentType instanceof ArrayTestType) { return resolveArray((List) argument, ((ArrayTestType) argumentType).getElementType()); diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericStdUDFWrapper.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericStdUDFWrapper.java index c610f984..f8ca23bd 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericStdUDFWrapper.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericStdUDFWrapper.java @@ -7,7 +7,6 @@ import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.StdUDF0; import com.linkedin.transport.api.udf.StdUDF1; @@ -24,6 +23,7 @@ import com.linkedin.transport.utils.FileSystemUtils; import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -42,7 +42,7 @@ public class GenericStdUDFWrapper { protected boolean _requiredFilesProcessed; protected StdFactory _stdFactory; private boolean[] _nullableArguments; - private StdData[] _args; + private Object[] _args; private Class _topLevelUdfClass; private List> _stdUdfImplementations; private String[] _localFiles; @@ -83,12 +83,18 @@ protected boolean containsNullValuedNonNullableArgument(Object[] arguments) { return false; } - protected StdData wrap(Object argument, StdData stdData) { - if (argument != null) { - ((PlatformData) stdData).setUnderlyingData(argument); - return stdData; - } else { + protected Object wrap(Object argument, Object stdData) { + if (argument == null) { return null; + } else { + if (argument instanceof Integer || argument instanceof Long || argument instanceof Boolean || + argument instanceof String || argument instanceof Double || argument instanceof Float || + argument instanceof ByteBuffer) { + return argument; + } else { + ((PlatformData) stdData).setUnderlyingData(argument); + return stdData; + } } } @@ -107,26 +113,26 @@ protected Class getTopLevelUdfClass() { } protected void createStdData() { - _args = new StdData[_inputTypes.length]; + _args = new Object[_inputTypes.length]; for (int i = 0; i < _inputTypes.length; i++) { _args[i] = GenericWrapper.createStdData(null, _inputTypes[i]); } } - private StdData[] wrapArguments(Object[] arguments) { - return IntStream.range(0, _args.length).mapToObj(i -> wrap(arguments[i], _args[i])).toArray(StdData[]::new); + private Object[] wrapArguments(Object[] arguments) { + return IntStream.range(0, _args.length).mapToObj(i -> wrap(arguments[i], _args[i])).toArray(Object[]::new); } public Object evaluate(Object[] arguments) { if (containsNullValuedNonNullableArgument(arguments)) { return null; } - StdData[] args = wrapArguments(arguments); + Object[] args = wrapArguments(arguments); if (!_requiredFilesProcessed) { String[] requiredFiles = getRequiredFiles(args); processRequiredFiles(requiredFiles); } - StdData result; + Object result; switch (args.length) { case 0: result = ((StdUDF0) _stdUdf).eval(); @@ -158,10 +164,10 @@ public Object evaluate(Object[] arguments) { default: throw new UnsupportedOperationException("eval not yet supported for StdUDF" + args.length); } - return result == null ? null : ((PlatformData) result).getUnderlyingData(); + return GenericWrapper.getPlatformData(result); } - public String[] getRequiredFiles(StdData[] args) { + public String[] getRequiredFiles(Object[] args) { String[] requiredFiles; switch (args.length) { case 0: @@ -196,7 +202,7 @@ public String[] getRequiredFiles(StdData[] args) { } _localFiles = Arrays.stream(requiredFiles).map(requiredFile -> { try { - return FileSystemUtils.resolveLatest(requiredFile, FileSystemUtils.getLocalFileSystem()); + return FileSystemUtils.resolveLatest(requiredFile); } catch (IOException e) { throw new RuntimeException("Failed to resolve path: [" + requiredFile + "].", e); } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericTester.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericTester.java index d3250c6e..f7a56525 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericTester.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericTester.java @@ -14,6 +14,7 @@ import com.linkedin.transport.test.spi.types.TestType; import com.linkedin.transport.typesystem.TypeSignature; import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,6 +52,12 @@ public void check(TestCase testCase) { Pair result = _executor.executeQuery(testCase.getFunctionCall()); Assert.assertEquals(result.getLeft(), _typeFactory.createType(TypeSignature.parse(testCase.getExpectedOutputType()), _boundVariables)); - Assert.assertEquals(result.getRight(), testCase.getExpectedOutput()); + if (testCase.getExpectedOutput() instanceof ByteBuffer) { + byte[] expected = ((ByteBuffer) testCase.getExpectedOutput()).array(); + byte[] actual = ((ByteBuffer) result.getRight()).array(); + Assert.assertEquals(actual, expected); + } else { + Assert.assertEquals(result.getRight(), testCase.getExpectedOutput()); + } } } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericWrapper.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericWrapper.java index 54b17bd1..195e3eae 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericWrapper.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/GenericWrapper.java @@ -5,18 +5,17 @@ */ package com.linkedin.transport.test.generic; -import com.linkedin.transport.api.data.StdData; +import com.linkedin.transport.api.data.PlatformData; import com.linkedin.transport.api.types.StdType; -import com.linkedin.transport.test.generic.data.GenericArray; -import com.linkedin.transport.test.generic.data.GenericBoolean; -import com.linkedin.transport.test.generic.data.GenericInteger; -import com.linkedin.transport.test.generic.data.GenericLong; -import com.linkedin.transport.test.generic.data.GenericMap; -import com.linkedin.transport.test.generic.data.GenericString; +import com.linkedin.transport.test.generic.data.GenericArrayData; +import com.linkedin.transport.test.generic.data.GenericMapData; import com.linkedin.transport.test.generic.data.GenericStruct; import com.linkedin.transport.test.spi.Row; import com.linkedin.transport.test.spi.types.ArrayTestType; import com.linkedin.transport.test.spi.types.BooleanTestType; +import com.linkedin.transport.test.spi.types.BinaryTestType; +import com.linkedin.transport.test.spi.types.DoubleTestType; +import com.linkedin.transport.test.spi.types.FloatTestType; import com.linkedin.transport.test.spi.types.IntegerTestType; import com.linkedin.transport.test.spi.types.LongTestType; import com.linkedin.transport.test.spi.types.MapTestType; @@ -24,6 +23,7 @@ import com.linkedin.transport.test.spi.types.StructTestType; import com.linkedin.transport.test.spi.types.TestType; import com.linkedin.transport.test.spi.types.UnknownTestType; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; @@ -33,21 +33,17 @@ public class GenericWrapper { private GenericWrapper() { } - public static StdData createStdData(Object data, TestType dataType) { + public static Object createStdData(Object data, TestType dataType) { if (dataType instanceof UnknownTestType) { return null; - } else if (dataType instanceof IntegerTestType) { - return new GenericInteger((Integer) data); - } else if (dataType instanceof LongTestType) { - return new GenericLong((Long) data); - } else if (dataType instanceof BooleanTestType) { - return new GenericBoolean((Boolean) data); - } else if (dataType instanceof StringTestType) { - return new GenericString((String) data); + } else if (dataType instanceof IntegerTestType || dataType instanceof LongTestType || + dataType instanceof FloatTestType || dataType instanceof DoubleTestType || + dataType instanceof BooleanTestType || dataType instanceof StringTestType || dataType instanceof BinaryTestType) { + return data; } else if (dataType instanceof ArrayTestType) { - return new GenericArray((List) data, dataType); + return new GenericArrayData((List) data, dataType); } else if (dataType instanceof MapTestType) { - return new GenericMap((Map) data, dataType); + return new GenericMapData((Map) data, dataType); } else if (dataType instanceof StructTestType) { return new GenericStruct((Row) data, dataType); } else { @@ -55,6 +51,20 @@ public static StdData createStdData(Object data, TestType dataType) { } } + public static Object getPlatformData(Object transportData) { + if (transportData == null) { + return null; + } else { + if (transportData instanceof Integer || transportData instanceof Long || transportData instanceof Float || + transportData instanceof Double || transportData instanceof Boolean || transportData instanceof ByteBuffer || + transportData instanceof String) { + return transportData; + } else { + return ((PlatformData) transportData).getUnderlyingData(); + } + } + } + public static StdType createStdType(TestType dataType) { return () -> dataType; } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArray.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArrayData.java similarity index 65% rename from transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArray.java rename to transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArrayData.java index b2152c93..2aa85cb9 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArray.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericArrayData.java @@ -5,9 +5,8 @@ */ package com.linkedin.transport.test.generic.data; +import com.linkedin.transport.api.data.ArrayData; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; import com.linkedin.transport.test.generic.GenericWrapper; import com.linkedin.transport.test.spi.types.ArrayTestType; import com.linkedin.transport.test.spi.types.TestType; @@ -15,12 +14,12 @@ import java.util.List; -public class GenericArray implements StdArray, PlatformData { +public class GenericArrayData implements ArrayData, PlatformData { private List _array; private TestType _elementType; - public GenericArray(List data, TestType type) { + public GenericArrayData(List data, TestType type) { _array = data; _elementType = ((ArrayTestType) type).getElementType(); } @@ -31,18 +30,18 @@ public int size() { } @Override - public StdData get(int idx) { - return GenericWrapper.createStdData(_array.get(idx), _elementType); + public E get(int idx) { + return (E) GenericWrapper.createStdData(_array.get(idx), _elementType); } @Override - public void add(StdData e) { - _array.add(((PlatformData) e).getUnderlyingData()); + public void add(E e) { + _array.add(GenericWrapper.getPlatformData(e)); } @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { private final Iterator _iterator = _array.iterator(); @Override @@ -51,8 +50,8 @@ public boolean hasNext() { } @Override - public StdData next() { - return GenericWrapper.createStdData(_iterator.next(), _elementType); + public E next() { + return (E) GenericWrapper.createStdData(_iterator.next(), _elementType); } }; } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericBoolean.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericBoolean.java deleted file mode 100644 index e731a1e3..00000000 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericBoolean.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.test.generic.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdBoolean; - - -public class GenericBoolean implements StdBoolean, PlatformData { - private Boolean _boolean; - - public GenericBoolean(Boolean aBoolean) { - _boolean = aBoolean; - } - - @Override - public boolean get() { - return _boolean; - } - - @Override - public Object getUnderlyingData() { - return _boolean; - } - - @Override - public void setUnderlyingData(Object value) { - _boolean = (Boolean) value; - } -} diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericInteger.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericInteger.java deleted file mode 100644 index bcb1905c..00000000 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericInteger.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.test.generic.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdInteger; - - -public class GenericInteger implements StdInteger, PlatformData { - private Integer _integer; - - public GenericInteger(Integer integer) { - _integer = integer; - } - - @Override - public int get() { - return _integer; - } - - @Override - public Object getUnderlyingData() { - return _integer; - } - - @Override - public void setUnderlyingData(Object value) { - _integer = (Integer) value; - } -} diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericLong.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericLong.java deleted file mode 100644 index 85e9dac6..00000000 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericLong.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.test.generic.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdLong; - - -public class GenericLong implements StdLong, PlatformData { - private Long _long; - - public GenericLong(Long aLong) { - _long = aLong; - } - - @Override - public long get() { - return _long; - } - - @Override - public Object getUnderlyingData() { - return _long; - } - - @Override - public void setUnderlyingData(Object value) { - _long = (Long) value; - } -} diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMap.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMapData.java similarity index 59% rename from transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMap.java rename to transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMapData.java index beeeb684..343fbac1 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMap.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericMapData.java @@ -5,9 +5,8 @@ */ package com.linkedin.transport.test.generic.data; +import com.linkedin.transport.api.data.MapData; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; import com.linkedin.transport.test.generic.GenericWrapper; import com.linkedin.transport.test.spi.types.MapTestType; import com.linkedin.transport.test.spi.types.TestType; @@ -20,19 +19,19 @@ import java.util.stream.Collectors; -public class GenericMap implements StdMap, PlatformData { +public class GenericMapData implements MapData, PlatformData { private Map _map; private final TestType _keyType; private final TestType _valueType; - public GenericMap(Map map, TestType type) { + public GenericMapData(Map map, TestType type) { _map = map; _keyType = ((MapTestType) type).getKeyType(); _valueType = ((MapTestType) type).getValueType(); } - public GenericMap(TestType type) { + public GenericMapData(TestType type) { this(new LinkedHashMap<>(), type); } @@ -52,21 +51,21 @@ public int size() { } @Override - public StdData get(StdData key) { - return GenericWrapper.createStdData(_map.get(((PlatformData) key).getUnderlyingData()), _valueType); + public V get(K key) { + return (V) GenericWrapper.createStdData(_map.get(GenericWrapper.getPlatformData(key)), _valueType); } @Override - public void put(StdData key, StdData value) { - _map.put(((PlatformData) key).getUnderlyingData(), ((PlatformData) value).getUnderlyingData()); + public void put(K key, V value) { + _map.put(GenericWrapper.getPlatformData(key), GenericWrapper.getPlatformData(value)); } @Override - public Set keySet() { - return new AbstractSet() { + public Set keySet() { + return new AbstractSet() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator keySet = _map.keySet().iterator(); @Override @@ -75,8 +74,8 @@ public boolean hasNext() { } @Override - public StdData next() { - return GenericWrapper.createStdData(keySet.next(), _keyType); + public K next() { + return (K) GenericWrapper.createStdData(keySet.next(), _keyType); } }; } @@ -89,12 +88,12 @@ public int size() { } @Override - public Collection values() { - return _map.values().stream().map(v -> GenericWrapper.createStdData(v, _valueType)).collect(Collectors.toList()); + public Collection values() { + return _map.values().stream().map(v -> (V) GenericWrapper.createStdData(v, _valueType)).collect(Collectors.toList()); } @Override - public boolean containsKey(StdData key) { - return _map.containsKey(((PlatformData) key).getUnderlyingData()); + public boolean containsKey(K key) { + return _map.containsKey(GenericWrapper.getPlatformData(key)); } } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericString.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericString.java deleted file mode 100644 index 4bb1babb..00000000 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericString.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.test.generic.data; - -import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdString; - - -public class GenericString implements StdString, PlatformData { - private String _string; - - public GenericString(String string) { - _string = string; - } - - @Override - public String get() { - return _string; - } - - @Override - public Object getUnderlyingData() { - return _string; - } - - @Override - public void setUnderlyingData(Object value) { - _string = (String) value; - } -} diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericStruct.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericStruct.java index e92b6043..333ddb32 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericStruct.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/data/GenericStruct.java @@ -6,8 +6,7 @@ package com.linkedin.transport.test.generic.data; import com.linkedin.transport.api.data.PlatformData; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.test.generic.GenericWrapper; import com.linkedin.transport.test.spi.Row; import com.linkedin.transport.test.spi.types.StructTestType; @@ -19,7 +18,7 @@ import java.util.stream.IntStream; -public class GenericStruct implements StdStruct, PlatformData { +public class GenericStruct implements RowData, PlatformData { private Row _struct; private final List _fieldNames; @@ -46,27 +45,27 @@ public void setUnderlyingData(Object value) { } @Override - public StdData getField(int index) { + public Object getField(int index) { return GenericWrapper.createStdData(_struct.getFields().get(index), _fieldTypes.get(index)); } @Override - public StdData getField(String name) { + public Object getField(String name) { return getField(_fieldNames.indexOf(name)); } @Override - public void setField(int index, StdData value) { - _struct.getFields().set(index, ((PlatformData) value).getUnderlyingData()); + public void setField(int index, Object value) { + _struct.getFields().set(index, GenericWrapper.getPlatformData(value)); } @Override - public void setField(String name, StdData value) { + public void setField(String name, Object value) { setField(_fieldNames.indexOf(name), value); } @Override - public List fields() { + public List fields() { return IntStream.range(0, _struct.getFields().size()).mapToObj(this::getField).collect(Collectors.toList()); } } diff --git a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/typesystem/GenericTypeSystem.java b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/typesystem/GenericTypeSystem.java index 0bae065b..148d6b02 100644 --- a/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/typesystem/GenericTypeSystem.java +++ b/transportable-udfs-test/transportable-udfs-test-generic/src/main/java/com/linkedin/transport/test/generic/typesystem/GenericTypeSystem.java @@ -7,6 +7,9 @@ import com.linkedin.transport.test.spi.types.ArrayTestType; import com.linkedin.transport.test.spi.types.BooleanTestType; +import com.linkedin.transport.test.spi.types.BinaryTestType; +import com.linkedin.transport.test.spi.types.DoubleTestType; +import com.linkedin.transport.test.spi.types.FloatTestType; import com.linkedin.transport.test.spi.types.IntegerTestType; import com.linkedin.transport.test.spi.types.LongTestType; import com.linkedin.transport.test.spi.types.MapTestType; @@ -66,6 +69,21 @@ protected boolean isStringType(TestType dataType) { return dataType instanceof StringTestType; } + @Override + protected boolean isFloatType(TestType dataType) { + return dataType instanceof FloatTestType; + } + + @Override + protected boolean isDoubleType(TestType dataType) { + return dataType instanceof DoubleTestType; + } + + @Override + protected boolean isBinaryType(TestType dataType) { + return dataType instanceof BinaryTestType; + } + @Override protected boolean isArrayType(TestType dataType) { return dataType instanceof ArrayTestType; @@ -101,6 +119,21 @@ protected TestType createStringType() { return TestTypeFactory.STRING_TEST_TYPE; } + @Override + protected TestType createFloatType() { + return TestTypeFactory.FLOAT_TEST_TYPE; + } + + @Override + protected TestType createDoubleType() { + return TestTypeFactory.DOUBLE_TEST_TYPE; + } + + @Override + protected TestType createBinaryType() { + return TestTypeFactory.BINARY_TEST_TYPE; + } + @Override protected TestType createUnknownType() { return TestTypeFactory.UNKNOWN_TEST_TYPE; diff --git a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/ToHiveTestOutputConverter.java b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/ToHiveTestOutputConverter.java index a4a2fe8a..77c88817 100644 --- a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/ToHiveTestOutputConverter.java +++ b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/ToHiveTestOutputConverter.java @@ -9,6 +9,7 @@ import com.linkedin.transport.test.spi.ToPlatformTestOutputConverter; import com.linkedin.transport.test.spi.types.StringTestType; import com.linkedin.transport.test.spi.types.TestType; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -50,6 +51,11 @@ public Object getStructData(Row struct, List fieldTypes, List .collect(Collectors.joining(",", "{", "}")); } + @Override + public Object getBinaryData(ByteBuffer value) { + return value.array(); + } + /** * In the output provided by {@link org.apache.hive.service.server.HiveServer2}, complex types are represented by * strings. So we need to return String values for primitives nested inside complex types. diff --git a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/udf/MapFromEntries.java b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/udf/MapFromEntries.java index 4a415fd8..4867fcb4 100644 --- a/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/udf/MapFromEntries.java +++ b/transportable-udfs-test/transportable-udfs-test-hive/src/main/java/com/linkedin/transport/test/hive/udf/MapFromEntries.java @@ -7,10 +7,9 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.StdFactory; -import com.linkedin.transport.api.data.StdArray; -import com.linkedin.transport.api.data.StdData; -import com.linkedin.transport.api.data.StdMap; -import com.linkedin.transport.api.data.StdStruct; +import com.linkedin.transport.api.data.ArrayData; +import com.linkedin.transport.api.data.MapData; +import com.linkedin.transport.api.data.RowData; import com.linkedin.transport.api.types.StdMapType; import com.linkedin.transport.api.udf.StdUDF1; import com.linkedin.transport.api.udf.TopLevelStdUDF; @@ -21,7 +20,7 @@ * Hive's built-in map() UDF cannot be used to create maps with complex key types. This UDF allows you to do so. * This is used inside {@link com.linkedin.transport.test.hive.HiveTester} to create arbitrary map objects */ -public class MapFromEntries extends StdUDF1 implements TopLevelStdUDF { +public class MapFromEntries extends StdUDF1 implements TopLevelStdUDF { private StdMapType _mapType; @@ -32,10 +31,10 @@ public void init(StdFactory stdFactory) { } @Override - public StdMap eval(StdArray entryArray) { - StdMap result = getStdFactory().createMap(_mapType); - for (StdData element : entryArray) { - StdStruct elementStruct = (StdStruct) element; + public MapData eval(ArrayData entryArray) { + MapData result = getStdFactory().createMap(_mapType); + for (Object element : entryArray) { + RowData elementStruct = (RowData) element; result.put(elementStruct.getField(0), elementStruct.getField(1)); } return result; diff --git a/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/PrestoSqlFunctionCallGenerator.java b/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/PrestoSqlFunctionCallGenerator.java index f626474c..01b26920 100644 --- a/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/PrestoSqlFunctionCallGenerator.java +++ b/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/PrestoSqlFunctionCallGenerator.java @@ -8,6 +8,7 @@ import com.linkedin.transport.test.spi.Row; import com.linkedin.transport.test.spi.SqlFunctionCallGenerator; import com.linkedin.transport.test.spi.types.TestType; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -16,6 +17,11 @@ public class PrestoSqlFunctionCallGenerator implements SqlFunctionCallGenerator { + @Override + public String getFloatArgumentString(Float value) { + return "REAL '" + value + "'"; + } + @Override public String getLongArgumentString(Long value) { return "CAST(" + String.valueOf(value) + " AS BIGINT)"; @@ -26,6 +32,12 @@ public String getStringArgumentString(String value) { return "CAST('" + String.valueOf(value) + "' AS VARCHAR)"; } + @Override + public String getBinaryArgumentString(ByteBuffer value) { + String base64EncodedValue = BASE64_ENCODER.encodeToString(value.array()); + return "from_base64('" + base64EncodedValue + "')"; + } + @Override public String getArrayArgumentString(List array, TestType arrayElementType) { return "ARRAY" + "[" + array.stream() diff --git a/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/ToPrestoTestOutputConverter.java b/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/ToPrestoTestOutputConverter.java index 4d11b332..204168d6 100644 --- a/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/ToPrestoTestOutputConverter.java +++ b/transportable-udfs-test/transportable-udfs-test-presto/src/main/java/com/linkedin/transport/test/presto/ToPrestoTestOutputConverter.java @@ -8,6 +8,8 @@ import com.linkedin.transport.test.spi.Row; import com.linkedin.transport.test.spi.ToPlatformTestOutputConverter; import com.linkedin.transport.test.spi.types.TestType; +import io.prestosql.spi.type.SqlVarbinary; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,4 +48,9 @@ public Object getStructData(Row struct, List fieldTypes, List .mapToObj(idx -> convertToTestOutput(struct.getFields().get(idx), fieldTypes.get(idx))) .collect(Collectors.toList()); } + + @Override + public Object getBinaryData(ByteBuffer value) { + return new SqlVarbinary(value.array()); + } } diff --git a/transportable-udfs-test/transportable-udfs-test-spark/build.gradle b/transportable-udfs-test/transportable-udfs-test-spark/build.gradle index 5d4bb833..d01ea53e 100644 --- a/transportable-udfs-test/transportable-udfs-test-spark/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-spark/build.gradle @@ -14,4 +14,5 @@ dependencies { } compile('com.fasterxml.jackson.module:jackson-module-scala_2.11:2.7.9') compile 'org.testng:testng:6.11' + compile 'org.slf4j:slf4j-simple:1.7.25' } \ No newline at end of file diff --git a/transportable-udfs-test/transportable-udfs-test-spark/src/main/scala/com/linkedin/transport/test/spark/ToSparkTestOutputConverter.scala b/transportable-udfs-test/transportable-udfs-test-spark/src/main/scala/com/linkedin/transport/test/spark/ToSparkTestOutputConverter.scala index 5394844f..9123d379 100644 --- a/transportable-udfs-test/transportable-udfs-test-spark/src/main/scala/com/linkedin/transport/test/spark/ToSparkTestOutputConverter.scala +++ b/transportable-udfs-test/transportable-udfs-test-spark/src/main/scala/com/linkedin/transport/test/spark/ToSparkTestOutputConverter.scala @@ -5,6 +5,7 @@ */ package com.linkedin.transport.test.spark +import java.nio.ByteBuffer import java.util import com.linkedin.transport.test.spi.{Row, ToPlatformTestOutputConverter} @@ -38,4 +39,6 @@ class ToSparkTestOutputConverter extends ToPlatformTestOutputConverter { new GenericRow(0.until(struct.getFields.size).map(i => convertToTestOutput( struct.getFields.get(i), fieldTypes.get(i))).toArray[Any]) } + + override def getBinaryData(value: ByteBuffer): AnyRef = value.array() } diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlFunctionCallGenerator.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlFunctionCallGenerator.java index 31d614d2..98914842 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlFunctionCallGenerator.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlFunctionCallGenerator.java @@ -6,7 +6,10 @@ package com.linkedin.transport.test.spi; import com.linkedin.transport.test.spi.types.ArrayTestType; +import com.linkedin.transport.test.spi.types.BinaryTestType; import com.linkedin.transport.test.spi.types.BooleanTestType; +import com.linkedin.transport.test.spi.types.DoubleTestType; +import com.linkedin.transport.test.spi.types.FloatTestType; import com.linkedin.transport.test.spi.types.IntegerTestType; import com.linkedin.transport.test.spi.types.LongTestType; import com.linkedin.transport.test.spi.types.MapTestType; @@ -14,6 +17,8 @@ import com.linkedin.transport.test.spi.types.StructTestType; import com.linkedin.transport.test.spi.types.TestType; import com.linkedin.transport.test.spi.types.UnknownTestType; +import java.nio.ByteBuffer; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -24,6 +29,7 @@ * Creates a SQL function call string for the given function name and the function arguments */ public interface SqlFunctionCallGenerator { + Base64.Encoder BASE64_ENCODER = Base64.getEncoder(); /** * Returns SQL function call string of the format {@code functionName(argument1, argument2, argument3, ...)} @@ -53,6 +59,12 @@ default String getFunctionCallArgumentString(Object argument, TestType argumentT return getBooleanArgumentString((Boolean) argument); } else if (argumentType instanceof StringTestType) { return getStringArgumentString((String) argument); + } else if (argumentType instanceof DoubleTestType) { + return getDoubleArgumentString((Double) argument); + } else if (argumentType instanceof FloatTestType) { + return getFloatArgumentString((Float) argument); + } else if (argumentType instanceof BinaryTestType) { + return getBinaryArgumentString((ByteBuffer) argument); } else if (argumentType instanceof ArrayTestType) { return getArrayArgumentString((List) argument, ((ArrayTestType) argumentType).getElementType()); } else if (argumentType instanceof MapTestType) { @@ -85,6 +97,19 @@ default String getStringArgumentString(String value) { return "'" + value + "'"; } + default String getDoubleArgumentString(Double value) { + return "CAST(" + value + " AS double)"; + } + + default String getFloatArgumentString(Float value) { + return "CAST(" + value + " AS float)"; + } + + default String getBinaryArgumentString(ByteBuffer value) { + String base64EncodedValue = BASE64_ENCODER.encodeToString(value.array()); + return "unbase64('" + base64EncodedValue + "')"; + } + /** * Returns a SQL string of the format {@code ARRAY(ele1, ele2, ele3, ...)} representing an array literal */ diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/ToPlatformTestOutputConverter.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/ToPlatformTestOutputConverter.java index 2a269f48..6f2fde5d 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/ToPlatformTestOutputConverter.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/ToPlatformTestOutputConverter.java @@ -7,6 +7,9 @@ import com.linkedin.transport.test.spi.types.ArrayTestType; import com.linkedin.transport.test.spi.types.BooleanTestType; +import com.linkedin.transport.test.spi.types.BinaryTestType; +import com.linkedin.transport.test.spi.types.DoubleTestType; +import com.linkedin.transport.test.spi.types.FloatTestType; import com.linkedin.transport.test.spi.types.IntegerTestType; import com.linkedin.transport.test.spi.types.LongTestType; import com.linkedin.transport.test.spi.types.MapTestType; @@ -14,6 +17,7 @@ import com.linkedin.transport.test.spi.types.StructTestType; import com.linkedin.transport.test.spi.types.TestType; import com.linkedin.transport.test.spi.types.UnknownTestType; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; @@ -35,6 +39,12 @@ default Object convertToTestOutput(Object data, TestType dataType) { return getBooleanData((Boolean) data); } else if (dataType instanceof StringTestType) { return getStringData((String) data); + } else if (dataType instanceof FloatTestType) { + return getFloatData((Float) data); + } else if (dataType instanceof DoubleTestType) { + return getDoubleData((Double) data); + } else if (dataType instanceof BinaryTestType) { + return getBinaryData((ByteBuffer) data); } else if (dataType instanceof ArrayTestType) { return getArrayData((List) data, ((ArrayTestType) dataType).getElementType()); } else if (dataType instanceof MapTestType) { @@ -68,6 +78,18 @@ default Object getStringData(String value) { return value; } + default Object getFloatData(Float value) { + return value; + } + + default Object getDoubleData(Double value) { + return value; + } + + default Object getBinaryData(ByteBuffer value) { + return value; + } + Object getArrayData(List array, TestType elementType); Object getMapData(Map map, TestType mapKeyType, TestType mapValueType); diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/BinaryTestType.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/BinaryTestType.java new file mode 100644 index 00000000..5d466eea --- /dev/null +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/BinaryTestType.java @@ -0,0 +1,9 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.test.spi.types; + +public class BinaryTestType implements TestType { +} diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/DoubleTestType.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/DoubleTestType.java new file mode 100644 index 00000000..696c270e --- /dev/null +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/DoubleTestType.java @@ -0,0 +1,9 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.test.spi.types; + +public class DoubleTestType implements TestType { +} diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/FloatTestType.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/FloatTestType.java new file mode 100644 index 00000000..dd717802 --- /dev/null +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/FloatTestType.java @@ -0,0 +1,9 @@ +/** + * Copyright 2018 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.test.spi.types; + +public class FloatTestType implements TestType { +} diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeFactory.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeFactory.java index d5843c93..3b7888a4 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeFactory.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeFactory.java @@ -14,6 +14,9 @@ public class TestTypeFactory { public static final TestType INTEGER_TEST_TYPE = new IntegerTestType(); public static final TestType LONG_TEST_TYPE = new LongTestType(); public static final TestType STRING_TEST_TYPE = new StringTestType(); + public static final TestType FLOAT_TEST_TYPE = new FloatTestType(); + public static final TestType DOUBLE_TEST_TYPE = new DoubleTestType(); + public static final TestType BINARY_TEST_TYPE = new BinaryTestType(); public static final TestType UNKNOWN_TEST_TYPE = new UnknownTestType(); private TestTypeFactory() { diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeUtils.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeUtils.java index 1a8d6c15..670e4aad 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeUtils.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/types/TestTypeUtils.java @@ -7,6 +7,7 @@ import com.linkedin.transport.test.spi.FunctionCall; import com.linkedin.transport.test.spi.Row; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.List; import java.util.Map; @@ -29,6 +30,12 @@ public static TestType inferTypeFromData(Object data) { return TestTypeFactory.BOOLEAN_TEST_TYPE; } else if (data instanceof String) { return TestTypeFactory.STRING_TEST_TYPE; + } else if (data instanceof Float) { + return TestTypeFactory.FLOAT_TEST_TYPE; + } else if (data instanceof Double) { + return TestTypeFactory.DOUBLE_TEST_TYPE; + } else if (data instanceof ByteBuffer) { + return TestTypeFactory.BINARY_TEST_TYPE; } else if (data instanceof List) { return TestTypeFactory.array(inferCollectionTypeFromData((List) data, "array elements")); } else if (data instanceof Map) { diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractBoundVariables.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractBoundVariables.java index b8d35331..83f4ee92 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractBoundVariables.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractBoundVariables.java @@ -49,6 +49,18 @@ private boolean isStringType(T dataType) { return _typeSystem.isStringType(dataType); } + private boolean isFloatType(T dataType) { + return _typeSystem.isFloatType(dataType); + } + + private boolean isDoubleType(T dataType) { + return _typeSystem.isDoubleType(dataType); + } + + private boolean isBinaryType(T dataType) { + return _typeSystem.isBinaryType(dataType); + } + private boolean isArrayType(T dataType) { return _typeSystem.isArrayType(dataType); } @@ -132,6 +144,21 @@ public boolean bind(TypeSignature typeSignature, T dataType) { typeMismatch = true; } break; + case FLOAT: + if (!isFloatType(dataType)) { + typeMismatch = true; + } + break; + case DOUBLE: + if (!isDoubleType(dataType)) { + typeMismatch = true; + } + break; + case BINARY: + if (!isBinaryType(dataType)) { + typeMismatch = true; + } + break; case UNKNOWN: if (!isUnknownType(dataType)) { typeMismatch = true; diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeFactory.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeFactory.java index 245faf01..2447109e 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeFactory.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeFactory.java @@ -35,6 +35,18 @@ private T createStringType() { return _typeSystem.createStringType(); } + private T createFloatType() { + return _typeSystem.createFloatType(); + } + + private T createDoubleType() { + return _typeSystem.createDoubleType(); + } + + private T createBinaryType() { + return _typeSystem.createBinaryType(); + } + private T createUnknownType() { return _typeSystem.createUnknownType(); } @@ -71,6 +83,12 @@ public T createType(TypeSignature typeSignatureTree, AbstractBoundVariables b return createLongType(); case STRING: return createStringType(); + case FLOAT: + return createFloatType(); + case DOUBLE: + return createDoubleType(); + case BINARY: + return createBinaryType(); case UNKNOWN: return createUnknownType(); default: diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeInference.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeInference.java index 7efc3b71..971bbbbf 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeInference.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeInference.java @@ -52,6 +52,18 @@ private boolean isStringType(T dataType) { return _typeSystem.isStringType(dataType); } + private boolean isFloatType(T dataType) { + return _typeSystem.isFloatType(dataType); + } + + private boolean isDoubleType(T dataType) { + return _typeSystem.isDoubleType(dataType); + } + + private boolean isBinaryType(T dataType) { + return _typeSystem.isBinaryType(dataType); + } + private boolean isArrayType(T dataType) { return _typeSystem.isArrayType(dataType); } @@ -138,6 +150,12 @@ private String dataTypeToString(T dataType) { return "bigint"; } else if (isStringType(dataType)) { return "varchar"; + } else if (isFloatType(dataType)) { + return "real"; + } else if (isDoubleType(dataType)) { + return "double"; + } else if (isBinaryType(dataType)) { + return "varbinary"; } else if (isUnknownType(dataType)) { return "unknown"; } else if (isArrayType(dataType)) { diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeSystem.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeSystem.java index a1b2318a..00c5c74a 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeSystem.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/AbstractTypeSystem.java @@ -28,6 +28,12 @@ public abstract class AbstractTypeSystem { protected abstract boolean isStringType(T dataType); + protected abstract boolean isFloatType(T dataType); + + protected abstract boolean isDoubleType(T dataType); + + protected abstract boolean isBinaryType(T dataType); + protected abstract boolean isArrayType(T dataType); protected abstract boolean isMapType(T dataType); @@ -42,6 +48,12 @@ public abstract class AbstractTypeSystem { protected abstract T createStringType(); + protected abstract T createFloatType(); + + protected abstract T createDoubleType(); + + protected abstract T createBinaryType(); + protected abstract T createUnknownType(); protected abstract T createArrayType(T elementType); diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/ConcreteTypeSignatureElement.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/ConcreteTypeSignatureElement.java index 8a39ac13..e1948583 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/ConcreteTypeSignatureElement.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/ConcreteTypeSignatureElement.java @@ -14,6 +14,9 @@ public enum ConcreteTypeSignatureElement implements TypeSignatureElement { INTEGER(false, 0), LONG(false, 0), STRING(false, 0), + FLOAT(false, 0), + DOUBLE(false, 0), + BINARY(false, 0), UNKNOWN(false, 0), ARRAY(false, 1), MAP(false, 2), diff --git a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/TypeSignature.java b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/TypeSignature.java index aeff40b7..23d3d730 100644 --- a/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/TypeSignature.java +++ b/transportable-udfs-type-system/src/main/java/com/linkedin/transport/typesystem/TypeSignature.java @@ -159,6 +159,15 @@ private static TypeSignatureElement getTypeSignatureElement(String currentBase) case "varchar": currentBaseElement = ConcreteTypeSignatureElement.STRING; break; + case "real": + currentBaseElement = ConcreteTypeSignatureElement.FLOAT; + break; + case "double": + currentBaseElement = ConcreteTypeSignatureElement.DOUBLE; + break; + case "varbinary": + currentBaseElement = ConcreteTypeSignatureElement.BINARY; + break; case "unknown": currentBaseElement = ConcreteTypeSignatureElement.UNKNOWN; break; diff --git a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestBoundVariables.java b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestBoundVariables.java index 8fb02762..4bffd501 100644 --- a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestBoundVariables.java +++ b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestBoundVariables.java @@ -22,6 +22,9 @@ public abstract class AbstractTestBoundVariables { final private T LONG = getTypeSystem().createLongType(); final private T INTEGER = getTypeSystem().createIntegerType(); final private T STRING = getTypeSystem().createStringType(); + final private T FLOAT = getTypeSystem().createFloatType(); + final private T DOUBLE = getTypeSystem().createDoubleType(); + final private T BINARY = getTypeSystem().createBinaryType(); final private T BOOLEAN = getTypeSystem().createBooleanType(); final private T NULL = getTypeSystem().createUnknownType(); @@ -89,12 +92,12 @@ public void testBoundVariables2() { "K" ), ImmutableList.of( - map(STRING, array(array(struct(BOOLEAN, STRING)))), + map(STRING, array(array(struct(BOOLEAN, STRING, FLOAT, DOUBLE, BINARY)))), STRING ), ImmutableMap.of( "K", STRING, - "V", array(struct(BOOLEAN, STRING)) + "V", array(struct(BOOLEAN, STRING, FLOAT, DOUBLE, BINARY)) ) ); } diff --git a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestTypeFactory.java b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestTypeFactory.java index cf26480d..7d1f7e38 100644 --- a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestTypeFactory.java +++ b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/AbstractTestTypeFactory.java @@ -19,6 +19,9 @@ public abstract class AbstractTestTypeFactory { final private T LONG = getTypeSystem().createLongType(); final private T INTEGER = getTypeSystem().createIntegerType(); final private T STRING = getTypeSystem().createStringType(); + final private T FLOAT = getTypeSystem().createFloatType(); + final private T DOUBLE = getTypeSystem().createDoubleType(); + final private T BINARY = getTypeSystem().createBinaryType(); final private T BOOLEAN = getTypeSystem().createBooleanType(); final private T NULL = getTypeSystem().createUnknownType(); @@ -55,6 +58,9 @@ public void testCreateTypePrimitives() { assertCreateType("boolean", BOOLEAN); assertCreateType("bigint", LONG); assertCreateType("varchar", STRING); + assertCreateType("real", FLOAT); + assertCreateType("double", DOUBLE); + assertCreateType("varbinary", BINARY); assertCreateType("unknown", NULL); } @@ -77,5 +83,7 @@ public void testCreateTypeStruct() { assertCreateType("row(arrField array(integer), strField varchar, mapField map(varchar,varchar), rowField row(integer))", struct(Arrays.asList("arrField", "strField", "mapField", "rowField"), array(INTEGER), STRING, map(STRING, STRING), struct(INTEGER)) ); + assertCreateType("row(integer, bigint, varchar, boolean, real, double, varbinary, unknown)", + struct(INTEGER, LONG, STRING, BOOLEAN, FLOAT, DOUBLE, BINARY, NULL)); } } \ No newline at end of file diff --git a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TestTypeSignature.java b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TestTypeSignature.java index 0dead879..8954610d 100644 --- a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TestTypeSignature.java +++ b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TestTypeSignature.java @@ -24,6 +24,12 @@ public void testTypeSignatureParse() { Assert.assertEquals(TypeSignature.parse("bigint"), LONG); + Assert.assertEquals(TypeSignature.parse("real"), FLOAT); + + Assert.assertEquals(TypeSignature.parse("double"), DOUBLE); + + Assert.assertEquals(TypeSignature.parse("varbinary"), BINARY); + Assert.assertEquals(TypeSignature.parse("array(bigint)"), array(LONG)); Assert.assertEquals(TypeSignature.parse("array(unknown)"), array(NULL)); @@ -31,8 +37,8 @@ public void testTypeSignatureParse() { Assert.assertEquals(TypeSignature.parse("array(map(varchar,boolean))"), array(map(STRING, BOOLEAN))); Assert.assertEquals( - TypeSignature.parse("array(row(varchar,boolean,integer))"), - array(struct(STRING, BOOLEAN, INTEGER))); + TypeSignature.parse("array(row(varchar,boolean,integer,real,double,varbinary))"), + array(struct(STRING, BOOLEAN, INTEGER, FLOAT, DOUBLE, BINARY))); } @Test diff --git a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TypeSignatureFactory.java b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TypeSignatureFactory.java index 5a4b6a1e..b3b6f7c2 100644 --- a/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TypeSignatureFactory.java +++ b/transportable-udfs-type-system/src/test/java/com/linkedin/transport/typesystem/TypeSignatureFactory.java @@ -17,6 +17,9 @@ private TypeSignatureFactory() { final public static TypeSignature INTEGER = new TypeSignature(ConcreteTypeSignatureElement.INTEGER, null); final public static TypeSignature LONG = new TypeSignature(ConcreteTypeSignatureElement.LONG, null); final public static TypeSignature STRING = new TypeSignature(ConcreteTypeSignatureElement.STRING, null); + final public static TypeSignature FLOAT = new TypeSignature(ConcreteTypeSignatureElement.FLOAT, null); + final public static TypeSignature DOUBLE = new TypeSignature(ConcreteTypeSignatureElement.DOUBLE, null); + final public static TypeSignature BINARY = new TypeSignature(ConcreteTypeSignatureElement.BINARY, null); final public static TypeSignature NULL = new TypeSignature(ConcreteTypeSignatureElement.UNKNOWN, null); public static TypeSignature array(TypeSignature elementTypeSignature) { diff --git a/transportable-udfs-utils/src/main/java/com/linkedin/transport/utils/FileSystemUtils.java b/transportable-udfs-utils/src/main/java/com/linkedin/transport/utils/FileSystemUtils.java index 6f56b898..79b7b3c8 100644 --- a/transportable-udfs-utils/src/main/java/com/linkedin/transport/utils/FileSystemUtils.java +++ b/transportable-udfs-utils/src/main/java/com/linkedin/transport/utils/FileSystemUtils.java @@ -14,48 +14,35 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; /** * This Utils class handles multiple utilities methods related with Hadoop FileSystem. * */ public class FileSystemUtils { - public static final String MAPREDUCE_FRAMEWORK_NAME = "mapreduce.framework.name"; - public static final String MAPRED_JOB_TRACKER = "mapred.job.tracker"; - public static final String LOCAL = "local"; private FileSystemUtils() { // Empty on purpose } /** - * Checks if the current UDF is running in local environment or something else. + * Get the FileSystem for the path * - * @param conf the Hadoop configuration - * @return true if it is in local mode + * @return the Path's FileSystem if we are not in local mode, local FileSystem if we are. */ - public static boolean isLocalEnvironment(Configuration conf) { - return conf.get(MAPREDUCE_FRAMEWORK_NAME, conf.get(MAPRED_JOB_TRACKER, LOCAL)).equals(LOCAL); + public static FileSystem getFileSystem(String filePath) { + return getFileSystem(filePath, new Configuration()); } /** - * Get the HDFS FileSystem - * - * @return the HDFS FileSystem if we are not in local mode, local FileSystem if we are. + * Same as {@link #getFileSystem(String)} but allows passing a {@link Configuration} used to resolve the path */ - public static FileSystem getHDFSFileSystem() { + public static FileSystem getFileSystem(String filePath, Configuration conf) { FileSystem fs; - JobConf conf = new JobConf(); try { - // Checks if currently we are in local mode, which is basically when running unit tests - if (isLocalEnvironment(conf)) { - fs = FileSystem.getLocal(conf); - } else { - fs = FileSystem.get(conf); - } + fs = new Path(filePath).getFileSystem(conf); } catch (IOException e) { - throw new RuntimeException("Failed to load the HDFS file system.", e); + throw new RuntimeException("Failed to load the file system for path: " + filePath, e); } return fs; @@ -87,16 +74,22 @@ public static FileSystem getLocalFileSystem() { * the same path. * * @param path the path to resolve - * @param fs the filesystem used to resolve the path * @return the resolved path * @throws IOException when the filesystem could not resolve the path */ - public static String resolveLatest(String path, FileSystem fs) throws IOException { + public static String resolveLatest(String path) throws IOException { + return resolveLatest(path, new Configuration()); + } + + /** + * Same as {@link #resolveLatest(String)} but allows passing a {@link Configuration} used to resolve the path + */ + public static String resolveLatest(String path, Configuration conf) throws IOException { if (!StringUtils.isBlank(path)) { path = path.trim(); String[] split = path.split("#LATEST"); String retval = split[0]; - + FileSystem fs = getFileSystem(path, conf); for (int i = 1; i < split.length; ++i) { retval = resolveLatestHelper(retval, fs, true) + split[i]; } diff --git a/transportable-udfs-utils/src/test/java/com/linkedin/transport/utils/FileSystemUtilsTest.java b/transportable-udfs-utils/src/test/java/com/linkedin/transport/utils/FileSystemUtilsTest.java index e8c37f95..56ef3426 100644 --- a/transportable-udfs-utils/src/test/java/com/linkedin/transport/utils/FileSystemUtilsTest.java +++ b/transportable-udfs-utils/src/test/java/com/linkedin/transport/utils/FileSystemUtilsTest.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Paths; -import org.apache.hadoop.fs.FileSystem; import org.testng.Assert; import org.testng.annotations.Test; @@ -18,20 +17,18 @@ public class FileSystemUtilsTest { @Test public void testResolveLatest() throws IOException, URISyntaxException { - FileSystem fs = FileSystemUtils.getLocalFileSystem(); - - String resourcePath = getPathForResource("root"); + String resourcePath = "file://" + getPathForResource("root"); // Test cases to resolve #LATEST - String filePath = FileSystemUtils.resolveLatest(resourcePath + "/2018/11/02.dat", fs); + String filePath = FileSystemUtils.resolveLatest(resourcePath + "/2018/11/02.dat"); Assert.assertTrue( - FileSystemUtils.resolveLatest(resourcePath + "/2018/11/02.dat", fs).endsWith("/root/2018/11/02.dat")); + FileSystemUtils.resolveLatest(resourcePath + "/2018/11/02.dat").endsWith("/root/2018/11/02.dat")); Assert.assertTrue( - FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/11/#LATEST", fs).endsWith("/root/2019/11/02.dat")); + FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/11/#LATEST").endsWith("/root/2019/11/02.dat")); Assert.assertTrue( - FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/#LATEST/#LATEST", fs).endsWith("/root/2019/12/02.dat")); + FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/#LATEST/#LATEST").endsWith("/root/2019/12/02.dat")); Assert.assertTrue( - FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/#LATEST", fs).endsWith("/root/2019/13.dat")); + FileSystemUtils.resolveLatest(resourcePath + "/#LATEST/#LATEST").endsWith("/root/2019/13.dat")); } private String getPathForResource(String resource) throws URISyntaxException { diff --git a/travis-build.sh b/travis-build.sh new file mode 100755 index 00000000..a7444d32 --- /dev/null +++ b/travis-build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# TravisCI calls this script to build and test the Transport code. +# Gradle commands that are specific to the release process are +# called directly from the Travis CI configuration file. +# The rationale for placing these commands in a separate script is +# to make it easier for contributors to run these checks before +# submitting a PR. + +set -e + +cd "$(dirname "$0")" + +./gradlew clean build -s +./gradlew -p transportable-udfs-examples clean build -s diff --git a/version.properties b/version.properties index 563e2243..1b2be12a 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ #Version of the produced binaries. This file is intended to be checked-in. #It will be automatically bumped by release automation. -version=0.0.45 -previousVersion=0.0.44 +version=0.0.62 +previousVersion=0.0.61