Text_Column.mid for in-memory and database backends (#14420)

(cherry picked from commit 51f07c1)

Author: GregoryTravis

CHANGELOG.md

@@ -76,6 +76,7 @@
 - [Read files into DuckDB both spatial and not.][14367]
 - [Implement Text_Column to_case for DB backends][14386]
 - [Implement bulk loading to DuckDB][14402]
+- [Implement `Text_Column.text_mid` for in-memory and database backends.][14420]
 - [Initial file writing from DuckDB][14421]
 
 [13769]: https://github.com/enso-org/enso/pull/13769
@@ -101,6 +102,7 @@
 [14367]: https://github.com/enso-org/enso/pull/14367
 [14386]: https://github.com/enso-org/enso/pull/14386
 [14402]: https://github.com/enso-org/enso/pull/14402
+[14420]: https://github.com/enso-org/enso/pull/14420
 [14421]: https://github.com/enso-org/enso/pull/14421
 
 #### Enso Language & Runtime

distribution/lib/Standard/Database/0.0.0-dev/src/Dialects/Dialect_Flag.enso

@@ -147,7 +147,7 @@ type Dialect_Flag
        private: true
        ---
        Specifies how the max size of the char type behaves after
-       text_left/text_right.
+       text_left, _right, and _mid.
        If True the original size is kept, if False the size is reset.
     Char_Max_Size_After_Substring_Kept
     ## ---

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso

@@ -494,7 +494,7 @@ base_dialect_operations =
     functions = [["COALESCE", make_function "COALESCE"], ["ROW_MIN", make_function "MIN"], ["ROW_MAX", make_function "MAX"]]
     agg = [fun "MAX", fun "MIN", fun "AVG", fun "SUM"]
     counts = [fun "COUNT", ["COUNT_ROWS", make_constant "COUNT(*)"]]
-    text = [is_empty, bin "LIKE", simple_equals_ignore_case, fold_case, make_case_sensitive, length]
+    text = [is_empty, bin "LIKE", simple_equals_ignore_case, fold_case, make_case_sensitive, length, ["SUBSTR", make_function "SUBSTR"]]
     nulls = [["IS_NULL", make_right_unary_op "IS NULL"], ["FILL_NULL", make_function "COALESCE"]]
     contains = [["IS_IN", make_is_in], ["IS_IN_COLUMN", make_is_in_column]]
     types = [simple_cast]

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/DB_Column_Implementation.enso

@@ -10,6 +10,7 @@ import Standard.Base.Internal.Rounding_Helpers
 from Standard.Base.Metadata.Widget import Text_Input
 from Standard.Base.Widget_Helpers import make_format_chooser
 
+import Standard.Table.Column.Rest_Of_String
 import Standard.Table.Fill_With.Fill_With
 import Standard.Table.Internal.Column_Naming_Helper.Column_Naming_Helper
 import Standard.Table.Internal.Date_Time_Helpers
@@ -342,6 +343,17 @@ type DB_Column_Implementation
             new_name = (naming_helper this_column).function_name "text_right" [this_column, n]
             make_binary_op this_column "RIGHT" n2 new_name
 
+    text_mid (this_column : Column & DB_Column) (start : Column | Any) (length : Column | Any) =
+        Value_Type.expect_integer start <|
+            case length of
+                _ : Rest_Of_String ->
+                    new_name = (naming_helper this_column).function_name "text_mid" [this_column, start]
+                    make_op this_column "SUBSTR" [start + 1] new_name
+                _ ->
+                    Value_Type.expect_integer length <|
+                        new_name = (naming_helper this_column).function_name "text_mid" [this_column, start, length]
+                        make_op this_column "SUBSTR" [start + 1, length] new_name
+
     contains (this_column : Column & DB_Column) (other : Column | Text | Any) case_sensitivity:Case_Sensitivity =
         new_name = (naming_helper this_column).function_name "contains" [this_column, other]
         make_text_case_op this_column "CONTAINS" other case_sensitivity new_name

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso

@@ -439,7 +439,7 @@ type Postgres_Dialect
    ---
 make_dialect_operations =
     cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"], ["PROPER", Base_Generator.make_function "INITCAP"]]
-    text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace], left, right, regex_match]+concat_ops+cases+trim_ops
+    text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace], left, right, ["SUBSTR", _make_mid], regex_match]+concat_ops+cases+trim_ops
     counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]]
     arith_extensions = [is_nan, is_inf, is_finite, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]]
     bool = [bool_or]
@@ -695,6 +695,22 @@ left = Base_Generator.lift_binary_op "LEFT" str-> n->
 right = Base_Generator.lift_binary_op "RIGHT" str-> n->
     SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))"
 
+## ---
+   private: true
+   ---
+private _make_mid arguments:Vector -> SQL_Builder =
+    case arguments.length of
+        2 ->
+            str = arguments.at 0
+            start = arguments.at 1
+            SQL_Builder.code "substr(" ++ str ++ ", (" ++ start ++ ")::integer)"
+        3 ->
+            str = arguments.at 0
+            start = arguments.at 1
+            length = arguments.at 2
+            SQL_Builder.code "substr(" ++ str ++ ", (" ++ start ++ ")::integer, (" ++ length ++ ")::integer)"
+        _ -> Error.throw (Illegal_Argument.Error "SUBSTR requires 2 or 3 arguments.")
+
 ## ---
    private: true
    ---

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso

@@ -252,7 +252,7 @@ operations_dict =
 
     always_boolean_ops = ["==", "!=", "EQUALS_IGNORE_CASE", ">=", "<=", "<", ">", "BETWEEN", "AND", "OR", "NOT", "IS_NULL", "IS_EMPTY", "LIKE", "IS_IN", "IS_IN_COLUMN", "STARTS_WITH", "ENDS_WITH", "CONTAINS", "BOOL_OR", "IS_INF", "IS_FINITE"]
     always_floating_ops = ["/", "MOD", "AVG", "STDDEV_POP", "STDDEV_SAMP", "ROUND"]
-    always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE", "LEFT", "RIGHT", "UPPER", "LOWER"]
+    always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE", "LEFT", "RIGHT", "SUBSTR", "UPPER", "LOWER"]
     always_integer_ops = ["COUNT", "COUNT_IS_NULL", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT_ROWS", "COUNT_OVER_PARTITION", "ROW_NUMBER", "ROW_NUMBER_IN_GROUP", "GROUP_NUMBER", "GROUP_NUMBER_EQUAL_COUNT", "LENGTH", "SIGNUM"]
     same_as_first = ["TRUNCATE", "CEIL", "FLOOR", "FIRST", "LAST", "ABS"]
     arithmetic_ops = ["ADD_NUMBER", "-", "*", "^", "%", "SUM"]

distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Dialect.enso

@@ -549,7 +549,7 @@ private _op_needs_to_materialize_null_checks op -> Boolean =
    ---
 make_dialect_operations =
     cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]]
-    text = [starts_with, contains, ends_with, like, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace], left, right]+concat_ops+cases+trim_ops
+    text = [starts_with, contains, ends_with, like, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace], left, right, ["SUBSTR", _make_mid]]+concat_ops+cases+trim_ops
     counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]]
     arith_extensions = [floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]]
     bool = [bool_or]
@@ -711,6 +711,22 @@ left = Base_Generator.lift_binary_op "LEFT" str-> n->
 right = Base_Generator.lift_binary_op "RIGHT" str-> n->
     SQL_Builder.code "right(" ++ str ++ ", CAST(" ++ n ++ " AS INT))"
 
+## ---
+   private: true
+   ---
+private _make_mid arguments:Vector -> SQL_Builder =
+    case arguments.length of
+        2 ->
+            str = arguments.at 0
+            start = arguments.at 1
+            SQL_Builder.code "substring(" ++ str ++ ", " ++ start ++ ")"
+        3 ->
+            str = arguments.at 0
+            start = arguments.at 1
+            length = arguments.at 2
+            SQL_Builder.code "substring(" ++ str ++ ", " ++ start ++ ", " ++ length ++ ")"
+        _ -> Error.throw (Illegal_Argument.Error "SUBSTRING requires 2 or 3 arguments.")
+
 ## ---
    private: true
    ---

distribution/lib/Standard/Table/0.0.0-dev/docs/api/Column.md

@@ -86,10 +86,11 @@
     - starts_with self other:(Standard.Table.Column.Column|Standard.Base.Data.Text.Text|Standard.Base.Any.Any) case_sensitivity:Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
     - take self range:(Standard.Base.Data.Index_Sub_Range.Index_Sub_Range|Standard.Base.Data.Range.Range|Standard.Base.Data.Numbers.Integer)= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
     - text_cleanse self remove:(Standard.Base.Data.Vector.Vector Standard.Base.Data.Text.Regex.Named_Pattern.Named_Pattern) -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
-    - text_left self n:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer) -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
+    - text_left self n:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer|Standard.Base.Nothing.Nothing)= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
     - text_length self -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
+    - text_mid self start:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer|Standard.Base.Nothing.Nothing) length:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer|Standard.Base.Nothing.Nothing|Standard.Table.Column.Rest_Of_String)= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
     - text_replace self term:(Standard.Base.Data.Text.Text|Standard.Base.Data.Text.Regex.Regex|Standard.Table.Column.Column)= new_text:(Standard.Base.Data.Text.Text|Standard.Table.Column.Column)= case_sensitivity:Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity= only_first:Standard.Base.Data.Boolean.Boolean= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
-    - text_right self n:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer) -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
+    - text_right self n:(Standard.Table.Column.Column|Standard.Base.Data.Numbers.Integer|Standard.Base.Nothing.Nothing)= -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
     - to_js_object self -> Standard.Base.Data.Json.JS_Object
     - to_table self -> (Standard.Table.Table.Table|Standard.Base.Any.Any)
     - to_text self -> Standard.Base.Data.Text.Text
@@ -101,8 +102,9 @@
     - zip self right:(Standard.Table.Column.Column|Standard.Table.Table.Table)= keep_unmatched:(Standard.Base.Data.Boolean.Boolean|Standard.Base.Data.Vector.Report_Unmatched)= right_prefix:Standard.Base.Data.Text.Text= on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior= -> Standard.Table.Table.Table
     - || self other:(Standard.Table.Column.Column|Standard.Base.Any.Any) -> (Standard.Table.Column.Column&Standard.Base.Any.Any)
 - default_date_period column:Standard.Base.Any.Any -> Standard.Base.Any.Any
-- default_row_limit_for_read column:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - Standard.Table.Column.Column.from that:Standard.Base.Data.Vector.Vector name:Standard.Base.Data.Text.Text= -> Standard.Table.Column.Column
+- type Rest_Of_String
+- default_row_limit_for_read column:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - Standard.Base.Data.Vector.Vector.from that:Standard.Table.Column.Column -> Standard.Base.Data.Vector.Vector
 - Standard.Base.Data.Vector.Vector.from that:Standard.Table.Table.Table -> Standard.Base.Data.Vector.Vector
 - Standard.Table.Column.Column.from that:Standard.Base.Data.Range.Range name:Standard.Base.Data.Text.Text= -> Standard.Table.Column.Column

distribution/lib/Standard/Table/0.0.0-dev/src/Column.enso

@@ -1251,7 +1251,7 @@ type Column
              example_text_length =
                 Examples.text_column_1.text_left 5
        ```
-    text_left self (n : Column | Integer) -> Column & Any =
+    text_left self (n : Column | Integer | Nothing = 1) -> Column & Any =
         self.implementation.text_left self n
 
     ## ---
@@ -1271,9 +1271,30 @@ type Column
              example_text_length =
                 Examples.text_column_1.text_right 5
        ```
-    text_right self (n : Column | Integer) -> Column & Any =
+    text_right self (n : Column | Integer | Nothing = 1) -> Column & Any =
         self.implementation.text_right self n
 
+    ## ---
+       group: Standard.Base.Text
+       icon: preparation
+       ---
+       Gets the characters starting at `start` and continuing for `length`
+       characters, for each element of the column.
+       In the Database backends, the default text mid method of the particular
+       database is used.
+       In the in-memory backend, this will give you the `length` graphemes of
+       the string, starting at `start`.
+
+       ## Examples
+       ### import Standard.Examples
+
+       ```
+             example_text_length =
+                Examples.text_column_1.text_mid 5
+       ```
+    text_mid self (start : Column | Integer | Nothing) (length : Column | Integer | Nothing | Rest_Of_String = Rest_Of_String) -> Column & Any =
+        self.implementation.text_mid self start length
+
     ## ---
        group: Standard.Base.Logical
        icon: preparation
@@ -2575,6 +2596,12 @@ type Column
     offset self n:Integer=-1 fill_with:Fill_With=..Nothing -> Column & Any =
         self.implementation.offset self n fill_with
 
+## ---
+   private: true
+   ---
+   Used to specify that the rest of the string is to be return.
+type Rest_Of_String
+
 ## ---
    private: true
    ---

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/In_Memory_Column_Implementation.enso

@@ -16,6 +16,7 @@ import Standard.Base.Internal.Rounding_Helpers
 from Standard.Base.Data.Index_Sub_Range import drop_helper, normalize_ranges, take_helper
 
 import project.Column.Column
+import project.Column.Rest_Of_String
 import project.Constants.Previous_Value
 import project.Data_Formatter.Data_Formatter
 import project.Fill_With.Fill_With
@@ -359,16 +360,61 @@ type In_Memory_Column_Implementation
         Value_Type.expect_text this_column <|
             apply_unary_operation this_column TextLengthOperation.INSTANCE
 
-    text_left (this_column : Column & In_Memory_Column) (n : Column | Integer) =
+    text_left (this_column : Column & In_Memory_Column) (n : Column | Integer | Nothing) =
         Value_Type.expect_text this_column <| Value_Type.expect_integer n <|
             new_name = naming_helper.function_name "text_left" [this_column, n]
             _apply_case_sensitive_text_operation this_column n ..Default TextPartOperation.LEFT (a -> b -> a.take b) new_name
 
-    text_right (this_column : Column & In_Memory_Column) (n : Column | Integer) =
+    text_right (this_column : Column & In_Memory_Column) (n : Column | Integer | Nothing) =
         Value_Type.expect_text this_column <| Value_Type.expect_integer n <|
             new_name = naming_helper.function_name "text_right" [this_column, n]
             _apply_case_sensitive_text_operation this_column n ..Default TextPartOperation.RIGHT (a -> b -> a.take (..Last b)) new_name
 
+    text_mid (this_column : Column & In_Memory_Column) (start : Column | Integer | Nothing) (length : Column | Integer | Nothing | Rest_Of_String) =
+        Value_Type.expect_text this_column <| Value_Type.expect_integer start <|
+            new_name = naming_helper.function_name "text_mid" [this_column, start, length]
+
+            # Returns Nothing if any argument is Nothing, and returns "" for start out-of-bounds or if length is negative.
+            do_substring s start length =
+                if s.is_nothing || start.is_nothing || length.is_nothing then Nothing else
+                    nonneg_start = if start < 0 then 0 else start
+                    case length of
+                        _ : Rest_Of_String ->
+                            if nonneg_start >= s.length then "" else s.substring nonneg_start
+                        _ : Integer ->
+                            if length < 0 || nonneg_start >= s.length then "" else s.substring nonneg_start length
+
+            if Nothing == start || Nothing == length then this_column . const Nothing . rename new_name else
+                case start of
+                    _ : Integer ->
+                        case length of
+                            _ : Integer ->
+                                _apply_unary_map this_column new_name (s -> do_substring s start length) expected_result_type=Value_Type.Char . cast this_column.value_type
+                            _ : Rest_Of_String ->
+                                _apply_unary_map this_column new_name (s -> do_substring s start length) expected_result_type=Value_Type.Char . cast this_column.value_type
+                            _ : Column ->
+                                Value_Type.expect_integer length <|
+                                    _apply_binary_map this_column (s -> l-> do_substring s start l) length new_name skip_nulls=False expected_result_type=Value_Type.Char . cast this_column.value_type
+                    _ : Column ->
+                        case length of
+                            _ : Integer ->
+                                _apply_binary_map this_column (s -> st-> do_substring s st length) start new_name skip_nulls=False expected_result_type=Value_Type.Char . cast this_column.value_type
+                            _ : Rest_Of_String ->
+                                _apply_binary_map this_column (s -> st-> do_substring s st length) start new_name skip_nulls=False expected_result_type=Value_Type.Char . cast this_column.value_type
+                            _ : Column ->
+                                Value_Type.expect_integer length <|
+                                    row_count = this_column.row_count
+                                    java_this = this_column.java_column
+                                    java_start = (start:In_Memory_Column).java_column
+                                    java_length = (length:In_Memory_Column).java_column
+
+                                    builder = make_string_builder row_count
+                                    0.up_to row_count . each i->
+                                        replaced = do_substring (java_this.getItem i) (java_start.getItem i) (java_length.getItem i)
+                                        builder.append replaced
+
+                                    In_Memory_Column.from_storage new_name builder.seal . cast this_column.value_type
+
     contains (this_column : Column & In_Memory_Column) (other : Column | Text | Any) case_sensitivity:Case_Sensitivity = Value_Type.expect_text this_column <| Value_Type.expect_text other <|
         new_name = naming_helper.function_name "contains" [this_column, other]
         _apply_case_sensitive_text_operation this_column other case_sensitivity TextPredicates.CONTAINS (a -> b -> a.contains b case_sensitivity) new_name