mirror of
https://github.com/apache/doris.git
synced 2026-11-06 18:26:11 +00:00
linrrarity
5fc3698fa5
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
1. `LargeInt` is mapped to arrow's `str` type, but no conversion is
performed in python_server which leads to an error due to the mismatch
between the final result and the expect return type(pa.str) after the
internal calculation is converted to int.
```text
Doris> CREATE FUNCTION py_largeint_inc(LARGEINT)
-> RETURNS LARGEINT
-> PROPERTIES (
-> "type"="PYTHON_UDF",
-> "symbol"="evaluate",
-> "runtime_version"="3.12.11",
-> "always_nullable"="true"
-> )
-> AS $$
-> def evaluate(x):
-> if x is None:
-> return None
-> return int(x) + 1
-> $$;
Query OK, 0 rows affected (0.024 sec)
Doris> SELECT py_largeint_inc(1);
ERROR 1105 (HY000): errCode = 2, detailMessage = (127.0.0.1)[RUNTIME_ERROR]Expected bytes, got a 'int' object. Detail: Python exception: Traceback (most recent call last):
File "pyarrow/_flight.pyx", line 2315, in pyarrow._flight._do_exchange
File "/mnt/disk7/linzhenqi/d1/doris/output/be/plugins/python_udf/python_server.py", line 2481, in do_exchange
self._handle_exchange_udf(python_udf_meta, reader, writer)
File "/mnt/disk7/linzhenqi/d1/doris/output/be/plugins/python_udf/python_server.py", line 2013, in _handle_exc
```
now:
```text
Doris> select py_largeint_inc(1);
+--------------------+
| py_largeint_inc(1) |
+--------------------+
| 2 |
+--------------------+
```
2. Udaf does not maintain metadata for special types
(IPv4/IPv6/LargeInt), causing the input_type to be passed and not
processed normally.
```text
Doris> CREATE AGGREGATE FUNCTION udaf_count_loopback_ipv6_inline(IPV6)
-> RETURNS BIGINT
-> PROPERTIES (
-> "type" = "PYTHON_UDF",
-> "symbol" = "CountLoopbackIPv6UDAF",
-> "runtime_version" = "3.9.18"
-> )
-> AS $$
-> class CountLoopbackIPv6UDAF:
-> def __init__(self):
-> self.count = 0
->
-> def accumulate(self, value):
-> if value is not None and value.is_loopback:
-> self.count += 1
->
-> def merge(self, other_state):
-> if other_state is not None:
-> self.count += other_state
->
-> def finish(self):
-> return self.count
->
-> @property
-> def aggregate_state(self):
-> return self.count
-> $$;
Query OK, 0 rows affected (0.004 sec)
Doris> SELECT udaf_count_loopback_ipv6_inline(ip_v6) as loopback_ipv6_count
-> FROM test_pythonudaf_ipv6_table;
ERROR 1105 (HY000): errCode = 2, detailMessage = (127.0.0.1)[INTERNAL_ERROR][INTERNAL_ERROR]ACCUMULATE operation failed for place_id=140343167410176
0# doris::PythonUDAFClient::accumulate(long, bool, arrow::RecordBatch const&, long, long) at ../src/common/status.h:500
1# doris::AggregatePythonUDAFData::add(long, doris::IColumn const**, long, long, std::vector<std::shared_ptr<doris::IDataType const>, std::allocator<std::shared_ptr<doris::IDataType const> > > const&) at ./be/build_RELEASE/../src/exprs/aggregate/aggregate_f
```
now
```text
Doris> SELECT udaf_count_loopback_ipv6_inline(ip_v6) as loopback_ipv6_count
-> FROM test_pythonudaf_ipv6_table;
+---------------------+
| loopback_ipv6_count |
+---------------------+
| 1 |
+---------------------+
```
250 lines
1.9 KiB
Plaintext
250 lines
1.9 KiB
Plaintext
-- This file is automatically generated. You should know what you did if you want to edit this
|
|
-- !split_string --
|
|
apple
|
|
banana
|
|
cherry
|
|
|
|
-- !generate_series --
|
|
1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
10
|
|
11
|
|
12
|
|
|
|
-- !generate_series_multiple --
|
|
1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
10
|
|
11
|
|
12
|
|
|
|
-- !running_sum --
|
|
10 10
|
|
20 20
|
|
30 30
|
|
40 40
|
|
|
|
-- !explode_json --
|
|
apple
|
|
banana
|
|
cherry
|
|
|
|
-- !top_n --
|
|
A 100 1
|
|
A 90 1
|
|
A 80 1
|
|
A 70 1
|
|
B 200 1
|
|
B 190 1
|
|
|
|
-- !duplicate --
|
|
Hello 1
|
|
Hello 2
|
|
Hello 3
|
|
|
|
-- !filter_positive --
|
|
1
|
|
3
|
|
7
|
|
|
|
-- !cartesian --
|
|
A X
|
|
A Y
|
|
A Z
|
|
B X
|
|
B Y
|
|
B Z
|
|
|
|
-- !all_filtered --
|
|
|
|
-- !mixed_filter --
|
|
2 -5
|
|
4 -3
|
|
|
|
-- !empty_input --
|
|
|
|
-- !nullable_true --
|
|
1 1
|
|
2 \N
|
|
3 3
|
|
4 \N
|
|
5 5
|
|
|
|
-- !non_nullable_false --
|
|
1 20
|
|
2 40
|
|
3 60
|
|
|
|
-- !nullable_with_nulls --
|
|
1 \N
|
|
2 \N
|
|
3 \N
|
|
4 \N
|
|
|
|
-- !non_nullable_with_nulls --
|
|
1 0
|
|
2 20
|
|
3 0
|
|
4 40
|
|
|
|
-- !default_nullable --
|
|
1 HELLO
|
|
2 \N
|
|
3 WORLD
|
|
4 \N
|
|
|
|
-- !multi_nullable --
|
|
1 a
|
|
1 b
|
|
1 c
|
|
2 \N
|
|
2 x
|
|
2 z
|
|
3 \N
|
|
3 \N
|
|
3 \N
|
|
|
|
-- !scalar_int --
|
|
1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
|
|
-- !scalar_string --
|
|
apple
|
|
banana
|
|
cherry
|
|
|
|
-- !mixed_style --
|
|
10
|
|
20
|
|
30
|
|
|
|
-- !return_scalar --
|
|
HELLO
|
|
|
|
-- !multi_field_check --
|
|
42 84
|
|
|
|
-- !outer_without --
|
|
1 20
|
|
5 30
|
|
|
|
-- !outer_with --
|
|
1 20
|
|
2 \N
|
|
3 \N
|
|
4 \N
|
|
5 30
|
|
|
|
-- !outer_string_without --
|
|
1 hello
|
|
1 world
|
|
5 single
|
|
|
|
-- !outer_string_with --
|
|
1 hello
|
|
1 world
|
|
2 \N
|
|
3 \N
|
|
4 \N
|
|
5 single
|
|
|
|
-- !outer_range_without --
|
|
1 1
|
|
1 2
|
|
1 3
|
|
5 1
|
|
|
|
-- !outer_range_with --
|
|
1 1
|
|
1 2
|
|
1 3
|
|
2 \N
|
|
3 \N
|
|
4 \N
|
|
5 1
|
|
|
|
-- !outer_multifield_without --
|
|
1 Alice 30
|
|
5 Charlie 25
|
|
|
|
-- !outer_multifield_with --
|
|
1 Alice 30
|
|
2 \N \N
|
|
3 \N \N
|
|
4 \N \N
|
|
5 Charlie 25
|
|
|
|
-- !outer_mixed_functions --
|
|
1 1 1
|
|
1 1 2
|
|
1 1 3
|
|
1 1 4
|
|
1 1 5
|
|
1 2 1
|
|
1 2 2
|
|
1 2 3
|
|
1 2 4
|
|
1 2 5
|
|
1 3 1
|
|
1 3 2
|
|
1 3 3
|
|
1 3 4
|
|
1 3 5
|
|
1 4 1
|
|
1 4 2
|
|
1 4 3
|
|
1 4 4
|
|
1 4 5
|
|
1 5 1
|
|
1 5 2
|
|
1 5 3
|
|
1 5 4
|
|
1 5 5
|
|
3 1 1
|
|
3 1 2
|
|
3 1 3
|
|
3 2 1
|
|
3 2 2
|
|
3 2 3
|
|
3 3 1
|
|
3 3 2
|
|
3 3 3
|
|
|
|
-- !outer_builtin_explode --
|
|
1 1
|
|
1 2
|
|
1 3
|
|
4 5
|
|
|
|
-- !outer_builtin_explode_outer --
|
|
1 1
|
|
1 2
|
|
1 3
|
|
2 \N
|
|
3 \N
|
|
4 5
|
|
|
|
-- !outer_doc_inner --
|
|
1 apple,banana
|
|
4 cherry
|
|
|
|
-- !outer_doc_outer --
|
|
1 apple,banana
|
|
2 \N
|
|
3 \N
|
|
4 cherry
|
|
|
|
-- !largeint_expand --
|
|
100
|
|
101
|
|
|