Preliminary work for a future change in outputting results that uses less memory

changelog.rst

@@ -10,6 +10,10 @@ Features:
 * Remove the ``...`` in the continuation prompt and use empty space instead. (Thanks: `Amjith Ramanujam`_)
 * Add \conninfo and handle more parameters with \c (issue #716) (Thanks: `François Pietka`_)
 
+Internal changes:
+-----------------
+* Preliminary work for a future change in outputting results that uses less memory. (Thanks: `Dick Marinus`_)
+
 Bug Fixes:
 ----------
 

pgcli/main.py

@@ -12,6 +12,7 @@ import shutil
 import functools
 import humanize
 import datetime as dt
+import itertools
 from time import time, sleep
 from codecs import open
 
@@ -1011,16 +1012,26 @@ def format_output(title, cur, headers, status, settings):
         headers = [case_function(utf8tounicode(x)) for x in headers]
         rows = list(cur)
         formatted = formatter.format_output(rows, headers, **output_kwargs)
-        first_line = formatted[:formatted.find('\n')]
+
+        if isinstance(formatted, (text_type)):
+            formatted = iter(formatted.splitlines())
+
+        first_line = next(formatted)
+        formatted = itertools.chain([first_line], formatted)
+
+        if max_width:
+            formatted = list(formatted)
 
         if (not expanded and max_width and len(first_line) > max_width and headers):
             formatted = formatter.format_output(
                 rows, headers, format_name='vertical', **output_kwargs)
+            if isinstance(formatted, (text_type)):
+                formatted = iter(formatted.splitlines())
 
-        output.append(formatted)
+        output = itertools.chain(output, formatted)
 
     if status:  # Only print the status if it's not None.
-        output.append(status)
+        output = itertools.chain(output, [status])
 
     return output
 

tests/features/steps/expanded.py

@@ -43,7 +43,6 @@ def step_see_data(context, which):
                 x | 1\r
                 y | 1.0\r
                 z | 1.0000\r
-                \r
                 SELECT 1\r
             '''),
             timeout=1)

tests/test_main.py

@@ -57,14 +57,14 @@ def test_format_output():
                             'test status', settings)
     expected = [
         'Title',
-        '+---------+---------+\n'
-        '| head1   | head2   |\n'
-        '|---------+---------|\n'
-        '| abc     | def     |\n'
+        '+---------+---------+',
+        '| head1   | head2   |',
+        '|---------+---------|',
+        '| abc     | def     |',
         '+---------+---------+',
         'test status'
     ]
-    assert results == expected
+    assert list(results) == expected
 
 
 def test_format_array_output(executor):
@@ -78,15 +78,15 @@ def test_format_array_output(executor):
     """
     results = run(executor, statement)
     expected = [
-        '+----------------+------------------------+--------------+\n'
-        '| bigint_array   | nested_numeric_array   | 配列         |\n'
-        '|----------------+------------------------+--------------|\n'
-        '| {1,2,3}        | {{1,2},{3,4}}          | {å,魚,текст} |\n'
-        '| {}             | <null>                 | {<null>}     |\n'
+        '+----------------+------------------------+--------------+',
+        '| bigint_array   | nested_numeric_array   | 配列         |',
+        '|----------------+------------------------+--------------|',
+        '| {1,2,3}        | {{1,2},{3,4}}          | {å,魚,текст} |',
+        '| {}             | <null>                 | {<null>}     |',
         '+----------------+------------------------+--------------+',
         'SELECT 2'
     ]
-    assert results == expected
+    assert list(results) == expected
 
 
 def test_format_array_output_expanded(executor):
@@ -100,17 +100,17 @@ def test_format_array_output_expanded(executor):
     """
     results = run(executor, statement, expanded=True)
     expected = [
-        '-[ RECORD 1 ]-------------------------\n'
-        'bigint_array         | {1,2,3}\n'
-        'nested_numeric_array | {{1,2},{3,4}}\n'
-        '配列                   | {å,魚,текст}\n'
-        '-[ RECORD 2 ]-------------------------\n'
-        'bigint_array         | {}\n'
-        'nested_numeric_array | <null>\n'
-        '配列                   | {<null>}\n',
+        '-[ RECORD 1 ]-------------------------',
+        'bigint_array         | {1,2,3}',
+        'nested_numeric_array | {{1,2},{3,4}}',
+        '配列                   | {å,魚,текст}',
+        '-[ RECORD 2 ]-------------------------',
+        'bigint_array         | {}',
+        'nested_numeric_array | <null>',
+        '配列                   | {<null>}',
         'SELECT 2'
     ]
-    assert results == expected
+    assert list(results) == expected
 
 
 def test_format_output_auto_expand():
@@ -120,14 +120,14 @@ def test_format_output_auto_expand():
                                   ['head1', 'head2'], 'test status', settings)
     table = [
         'Title',
-        '+---------+---------+\n'
-        '| head1   | head2   |\n'
-        '|---------+---------|\n'
-        '| abc     | def     |\n'
+        '+---------+---------+',
+        '| head1   | head2   |',
+        '|---------+---------|',
+        '| abc     | def     |',
         '+---------+---------+',
         'test status'
     ]
-    assert table_results == table
+    assert list(table_results) == table
     expanded_results = format_output(
         'Title',
         [('abc', 'def')],
@@ -137,12 +137,12 @@ def test_format_output_auto_expand():
     )
     expanded = [
         'Title',
-        '-[ RECORD 1 ]-------------------------\n'
-        'head1 | abc\n'
-        'head2 | def\n',
+        '-[ RECORD 1 ]-------------------------',
+        'head1 | abc',
+        'head2 | def',
         'test status'
     ]
-    assert expanded_results == expanded
+    assert list(expanded_results) == expanded
 
 
 @dbtest

tests/test_pgexecute.py

@@ -181,25 +181,25 @@ def test_unicode_support_in_output(executor, expanded):
 @dbtest
 def test_multiple_queries_same_line(executor):
     result = run(executor, "select 'foo'; select 'bar'")
-    assert len(result) == 4  # 2 * (output+status)
-    assert "foo" in result[0]
-    assert "bar" in result[2]
+    assert len(result) == 12  # 2 * (output+status) * 3 lines
+    assert "foo" in result[3]
+    assert "bar" in result[9]
 
 
 @dbtest
 def test_multiple_queries_with_special_command_same_line(executor, pgspecial):
     result = run(executor, "select 'foo'; \d", pgspecial=pgspecial)
-    assert len(result) == 4  # 2 * (output+status)
-    assert "foo" in result[0]
+    assert len(result) == 11  # 2 * (output+status) * 3 lines
+    assert "foo" in result[3]
     # This is a lame check. :(
-    assert "Schema" in result[2]
+    assert "Schema" in result[7]
 
 
 @dbtest
 def test_multiple_queries_same_line_syntaxerror(executor, exception_formatter):
     result = run(executor, u"select 'fooé'; invalid syntax é",
                  exception_formatter=exception_formatter)
-    assert u'fooé' in result[0]
+    assert u'fooé' in result[3]
     assert 'syntax error at or near "invalid"' in result[-1]
 
 
@@ -210,9 +210,9 @@ def pgspecial():
 
 @dbtest
 def test_special_command_help(executor, pgspecial):
-    result = run(executor, '\\?', pgspecial=pgspecial)[0].split('|')
-    assert(result[1].find(u'Command') != -1)
-    assert(result[2].find(u'Description') != -1)
+    result = run(executor, '\\?', pgspecial=pgspecial)[1].split('|')
+    assert u'Command' in result[1]
+    assert u'Description' in result[2]
 
 
 @dbtest