!24 Add map_reduce function
From: @zhang-liang-pengkun Reviewed-by: @Charlie_li Signed-off-by: @Charlie_li
This commit is contained in:
commit
fb3c71ef77
166
0003-Add-map_reduce-function-196.patch
Normal file
166
0003-Add-map_reduce-function-196.patch
Normal file
@ -0,0 +1,166 @@
|
|||||||
|
From edfb4704efe9bdb2dc6022fb4b665c62a1525a2c Mon Sep 17 00:00:00 2001
|
||||||
|
From: Bo Bayles <bbayles@gmail.com>
|
||||||
|
Date: Tue, 20 Feb 2018 06:04:55 -0600
|
||||||
|
Subject: [PATCH] Add map_reduce function (#196)
|
||||||
|
|
||||||
|
* Add map_reduce function
|
||||||
|
|
||||||
|
* Note return type
|
||||||
|
|
||||||
|
* keyfunc won't be None
|
||||||
|
|
||||||
|
* Correct typo in docstring
|
||||||
|
---
|
||||||
|
docs/api.rst | 1 +
|
||||||
|
more_itertools/more.py | 68 +++++++++++++++++++++++++++++++
|
||||||
|
more_itertools/tests/test_more.py | 35 +++++++++++++++-
|
||||||
|
3 files changed, 103 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/docs/api.rst b/docs/api.rst
|
||||||
|
index b1c0632..914e5bc 100644
|
||||||
|
--- a/docs/api.rst
|
||||||
|
+++ b/docs/api.rst
|
||||||
|
@@ -129,6 +129,7 @@ These tools return summarized or aggregated data from an iterable.
|
||||||
|
.. autofunction:: consecutive_groups(iterable, ordering=lambda x: x)
|
||||||
|
.. autofunction:: exactly_n(iterable, n, predicate=bool)
|
||||||
|
.. autoclass:: run_length
|
||||||
|
+.. autofunction:: map_reduce
|
||||||
|
|
||||||
|
----
|
||||||
|
|
||||||
|
diff --git a/more_itertools/more.py b/more_itertools/more.py
|
||||||
|
index 21a1497..c4b08ae 100644
|
||||||
|
--- a/more_itertools/more.py
|
||||||
|
+++ b/more_itertools/more.py
|
||||||
|
@@ -51,6 +51,7 @@ __all__ = [
|
||||||
|
'locate',
|
||||||
|
'lstrip',
|
||||||
|
'make_decorator',
|
||||||
|
+ 'map_reduce',
|
||||||
|
'numeric_range',
|
||||||
|
'one',
|
||||||
|
'padded',
|
||||||
|
@@ -1983,3 +1984,70 @@ def make_decorator(wrapping_func, result_index=0):
|
||||||
|
return outer_wrapper
|
||||||
|
|
||||||
|
return decorator
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None):
|
||||||
|
+ """Return a dictionary that maps the items in *iterable* to categories
|
||||||
|
+ defined by *keyfunc*, transforms them with *valuefunc*, and
|
||||||
|
+ then summarizes them by category with *reducefunc*.
|
||||||
|
+
|
||||||
|
+ *valuefunc* defaults to the identity function if it is unspecified.
|
||||||
|
+ If *reducefunc* is unspecified, no summarization takes place:
|
||||||
|
+
|
||||||
|
+ >>> keyfunc = lambda x: x.upper()
|
||||||
|
+ >>> result = map_reduce('abbccc', keyfunc)
|
||||||
|
+ >>> sorted(result.items())
|
||||||
|
+ [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])]
|
||||||
|
+
|
||||||
|
+ Specifying *valuefunc* transforms the categorized items:
|
||||||
|
+
|
||||||
|
+ >>> keyfunc = lambda x: x.upper()
|
||||||
|
+ >>> valuefunc = lambda x: 1
|
||||||
|
+ >>> result = map_reduce('abbccc', keyfunc, valuefunc)
|
||||||
|
+ >>> sorted(result.items())
|
||||||
|
+ [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])]
|
||||||
|
+
|
||||||
|
+ Specifying *reducefunc* summarizes the categorized items:
|
||||||
|
+
|
||||||
|
+ >>> keyfunc = lambda x: x.upper()
|
||||||
|
+ >>> valuefunc = lambda x: 1
|
||||||
|
+ >>> reducefunc = sum
|
||||||
|
+ >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc)
|
||||||
|
+ >>> sorted(result.items())
|
||||||
|
+ [('A', 1), ('B', 2), ('C', 3)]
|
||||||
|
+
|
||||||
|
+ You may want to filter the input iterable before applying the map/reduce
|
||||||
|
+ proecdure:
|
||||||
|
+
|
||||||
|
+ >>> all_items = range(30)
|
||||||
|
+ >>> items = [x for x in all_items if 10 <= x <= 20] # Filter
|
||||||
|
+ >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1
|
||||||
|
+ >>> categories = map_reduce(items, keyfunc=keyfunc)
|
||||||
|
+ >>> sorted(categories.items())
|
||||||
|
+ [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])]
|
||||||
|
+ >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum)
|
||||||
|
+ >>> sorted(summaries.items())
|
||||||
|
+ [(0, 90), (1, 75)]
|
||||||
|
+
|
||||||
|
+ Note that all items in the iterable are gathered into a list before the
|
||||||
|
+ summarization step, which may require significant storage.
|
||||||
|
+
|
||||||
|
+ The returned object is a :obj:`collections.defaultdict` with the
|
||||||
|
+ ``default_factory`` set to ``None``, such that it behaves like a normal
|
||||||
|
+ dictionary.
|
||||||
|
+
|
||||||
|
+ """
|
||||||
|
+ valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc
|
||||||
|
+
|
||||||
|
+ ret = defaultdict(list)
|
||||||
|
+ for item in iterable:
|
||||||
|
+ key = keyfunc(item)
|
||||||
|
+ value = valuefunc(item)
|
||||||
|
+ ret[key].append(value)
|
||||||
|
+
|
||||||
|
+ if reducefunc is not None:
|
||||||
|
+ for key, value_list in ret.items():
|
||||||
|
+ ret[key] = reducefunc(value_list)
|
||||||
|
+
|
||||||
|
+ ret.default_factory = None
|
||||||
|
+ return ret
|
||||||
|
diff --git a/more_itertools/tests/test_more.py b/more_itertools/tests/test_more.py
|
||||||
|
index d8ada9c..0ac4abd 100644
|
||||||
|
--- a/more_itertools/tests/test_more.py
|
||||||
|
+++ b/more_itertools/tests/test_more.py
|
||||||
|
@@ -15,7 +15,7 @@ from itertools import (
|
||||||
|
product,
|
||||||
|
repeat,
|
||||||
|
)
|
||||||
|
-from operator import add, itemgetter
|
||||||
|
+from operator import add, mul, itemgetter
|
||||||
|
from unittest import TestCase
|
||||||
|
|
||||||
|
from six.moves import filter, map, range, zip
|
||||||
|
@@ -1792,3 +1792,36 @@ class MakeDecoratorTests(TestCase):
|
||||||
|
|
||||||
|
it.seek(0)
|
||||||
|
self.assertEqual(list(it), ['0', '1', '2', '3', '4'])
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+class MapReduceTests(TestCase):
|
||||||
|
+ def test_default(self):
|
||||||
|
+ iterable = (str(x) for x in range(5))
|
||||||
|
+ keyfunc = lambda x: int(x) // 2
|
||||||
|
+ actual = sorted(mi.map_reduce(iterable, keyfunc).items())
|
||||||
|
+ expected = [(0, ['0', '1']), (1, ['2', '3']), (2, ['4'])]
|
||||||
|
+ self.assertEqual(actual, expected)
|
||||||
|
+
|
||||||
|
+ def test_valuefunc(self):
|
||||||
|
+ iterable = (str(x) for x in range(5))
|
||||||
|
+ keyfunc = lambda x: int(x) // 2
|
||||||
|
+ valuefunc = int
|
||||||
|
+ actual = sorted(mi.map_reduce(iterable, keyfunc, valuefunc).items())
|
||||||
|
+ expected = [(0, [0, 1]), (1, [2, 3]), (2, [4])]
|
||||||
|
+ self.assertEqual(actual, expected)
|
||||||
|
+
|
||||||
|
+ def test_reducefunc(self):
|
||||||
|
+ iterable = (str(x) for x in range(5))
|
||||||
|
+ keyfunc = lambda x: int(x) // 2
|
||||||
|
+ valuefunc = int
|
||||||
|
+ reducefunc = lambda value_list: reduce(mul, value_list, 1)
|
||||||
|
+ actual = sorted(
|
||||||
|
+ mi.map_reduce(iterable, keyfunc, valuefunc, reducefunc).items()
|
||||||
|
+ )
|
||||||
|
+ expected = [(0, 0), (1, 6), (2, 4)]
|
||||||
|
+ self.assertEqual(actual, expected)
|
||||||
|
+
|
||||||
|
+ def test_ret(self):
|
||||||
|
+ d = mi.map_reduce([1, 0, 2, 0, 1, 0], bool)
|
||||||
|
+ self.assertEqual(d, {False: [0, 0, 0], True: [1, 2, 1]})
|
||||||
|
+ self.assertRaises(KeyError, lambda: d[None].append(1))
|
||||||
|
--
|
||||||
|
2.39.0.windows.2
|
||||||
|
|
||||||
@ -1,6 +1,6 @@
|
|||||||
Name: python-more-itertools
|
Name: python-more-itertools
|
||||||
Version: 4.1.0
|
Version: 4.1.0
|
||||||
Release: 7
|
Release: 8
|
||||||
Summary: An opensource python library wrapping around itertools
|
Summary: An opensource python library wrapping around itertools
|
||||||
License: MIT
|
License: MIT
|
||||||
URL: https://github.com/erikrose/more-itertools
|
URL: https://github.com/erikrose/more-itertools
|
||||||
@ -9,6 +9,7 @@ BuildArch: noarch
|
|||||||
Patch0000: python37.patch
|
Patch0000: python37.patch
|
||||||
Patch0001: 0001-fix-for-python-3.7-deprecation-warning-about-importi.patch
|
Patch0001: 0001-fix-for-python-3.7-deprecation-warning-about-importi.patch
|
||||||
Patch0002: 0002-more.py-document-ilen.patch
|
Patch0002: 0002-more.py-document-ilen.patch
|
||||||
|
Patch0003: 0003-Add-map_reduce-function-196.patch
|
||||||
|
|
||||||
%description
|
%description
|
||||||
This is a python library for efficient use of itertools utility, which also \
|
This is a python library for efficient use of itertools utility, which also \
|
||||||
@ -63,6 +64,9 @@ See https://pythonhosted.org/more-itertools/index.html for more information.
|
|||||||
%{python3_sitelib}/more_itertools-%{version}-py%{python3_version}.egg-info
|
%{python3_sitelib}/more_itertools-%{version}-py%{python3_version}.egg-info
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Fri Dec 22 2023 zhangliangpengkun<zhangliangpengkun@xfusion.com> - 4.1.0-8
|
||||||
|
- Add map_reduce function (#196)
|
||||||
|
|
||||||
* Thu Nov 30 2023 zhangliangpengkun<zhangliangpengkun@xfusion.com> - 4.1.0-7
|
* Thu Nov 30 2023 zhangliangpengkun<zhangliangpengkun@xfusion.com> - 4.1.0-7
|
||||||
- more.py: document ilen
|
- more.py: document ilen
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user