Add a log extension

The log extension is responsible for retrieving logs from the system,
if journalctl is present the logs will come from it, otherwise we
fallback to getting the logs from the /var/log directory + dmesg logs.

In the coreos ramdisk, we need to bind mount /run/log in the container
so the IPA service can have access to the journal.

For the tinyIPA ramdisk, the logs from IPA are now being redirected to
/var/logs/ironic-python-agent.log instead of only going to the default
stdout.

Inspector now shares the same method of collecting logs, extending its
capabilities for non-systemd systems.

Partial-Bug: #1587143
Change-Id: Ie507e2e5c58cffa255bbfb2fa5ffb95cb98ed8c4

3 files changed, 164 insertions, 33 deletions

diff --git a/ironic_python_agent/tests/unit/extensions/test_log.py b/ironic_python_agent/tests/unit/extensions/test_log.py
new file mode 100644
index 00000000..ff90b914
--- /dev/null
+++ b/ironic_python_agent/tests/unit/extensions/test_log.py
@@ -0,0 +1,38 @@
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from ironic_python_agent import utils
+from oslotest import base as test_base
+
+from ironic_python_agent.extensions import log
+
+
+class TestLogExtension(test_base.BaseTestCase):
+
+    def setUp(self):
+        super(TestLogExtension, self).setUp()
+        self.agent_extension = log.LogExtension()
+
+    @mock.patch.object(utils, 'collect_system_logs', autospec=True)
+    def test_collect_system_logs(self, mock_collect):
+        ret = 'Squidward Tentacles'
+        mock_collect.return_value = ret
+
+        cmd_result = self.agent_extension.collect_system_logs()
+        serialized_cmd_result = cmd_result.serialize()
+        expected_ret = {'system_logs': ret}
+        self.assertEqual(expected_ret, serialized_cmd_result['command_result'])
diff --git a/ironic_python_agent/tests/unit/test_inspector.py b/ironic_python_agent/tests/unit/test_inspector.py
index dfeb3a67..7f52ac9f 100644
--- a/ironic_python_agent/tests/unit/test_inspector.py
+++ b/ironic_python_agent/tests/unit/test_inspector.py
@@ -13,12 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
 import collections
 import copy
-import io
 import os
-import tarfile
 import time
 
 import mock
@@ -26,7 +23,6 @@ from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslotest import base as test_base
 import requests
-import six
 import stevedore
 
 from ironic_python_agent import errors
@@ -379,41 +375,22 @@ class TestCollectDefault(BaseDiscoverTest):
         mock_wait_for_dhcp.assert_called_once_with()
 
 
-@mock.patch.object(utils, 'execute', autospec=True)
+@mock.patch.object(utils, 'collect_system_logs', autospec=True)
 class TestCollectLogs(test_base.BaseTestCase):
-    def test(self, mock_execute):
-        contents = 'journal contents \xd0\xbc\xd1\x8f\xd1\x83'
-        # That's how execute() works with binary=True
-        if six.PY3:
-            contents = b'journal contents \xd0\xbc\xd1\x8f\xd1\x83'
-        else:
-            contents = 'journal contents \xd0\xbc\xd1\x8f\xd1\x83'
-        expected_contents = u'journal contents \u043c\u044f\u0443'
-        mock_execute.return_value = (contents, '')
 
+    def test(self, mock_collect):
         data = {}
-        with mock.patch('time.time', return_value=42):
-            inspector.collect_logs(data, None)
-        res = io.BytesIO(base64.b64decode(data['logs']))
-
-        with tarfile.open(fileobj=res) as tar:
-            members = [(m.name, m.size, m.mtime) for m in tar]
-            self.assertEqual([('journal', len(contents), 42)], members)
-
-            member = tar.extractfile('journal')
-            self.assertEqual(expected_contents, member.read().decode('utf-8'))
+        ret = 'SpongeBob SquarePants'
+        mock_collect.return_value = ret
 
-        mock_execute.assert_called_once_with('journalctl', '--full',
-                                             '--no-pager', '-b',
-                                             '-n', '10000', binary=True,
-                                             log_stdout=False)
-
-    def test_no_journal(self, mock_execute):
-        mock_execute.side_effect = OSError()
+        inspector.collect_logs(data, None)
+        self.assertEqual({'logs': ret}, data)
 
+    def test_fail(self, mock_collect):
         data = {}
-        inspector.collect_logs(data, None)
-        self.assertFalse(data)
+        mock_collect.side_effect = errors.CommandExecutionError('boom')
+        self.assertIsNone(inspector.collect_logs(data, None))
+        self.assertNotIn('logs', data)
 
 
 @mock.patch.object(utils, 'execute', autospec=True)
diff --git a/ironic_python_agent/tests/unit/test_utils.py b/ironic_python_agent/tests/unit/test_utils.py
index be1626c7..aac438c9 100644
--- a/ironic_python_agent/tests/unit/test_utils.py
+++ b/ironic_python_agent/tests/unit/test_utils.py
@@ -13,10 +13,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import base64
 import errno
 import glob
+import io
 import os
 import shutil
+import subprocess
+import tarfile
 import tempfile
 import testtools
 
@@ -426,3 +430,115 @@ class TestFailures(testtools.TestCase):
         self.assertIsNone(f.raise_if_needed())
         f.add('foo')
         self.assertRaisesRegex(FakeException, 'foo', f.raise_if_needed)
+
+
+class TestUtils(testtools.TestCase):
+
+    def _get_journalctl_output(self, mock_execute, lines=None, units=None):
+        contents = b'Krusty Krab'
+        mock_execute.return_value = (contents, '')
+        data = utils.get_journalctl_output(lines=lines, units=units)
+
+        cmd = ['journalctl', '--full', '--no-pager', '-b']
+        if lines is not None:
+            cmd.extend(['-n', str(lines)])
+        if units is not None:
+            [cmd.extend(['-u', u]) for u in units]
+
+        mock_execute.assert_called_once_with(*cmd, binary=True,
+                                             log_stdout=False)
+        self.assertEqual(contents, data.read())
+
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_journalctl_output(self, mock_execute):
+        self._get_journalctl_output(mock_execute)
+
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_journalctl_output_with_lines(self, mock_execute):
+        self._get_journalctl_output(mock_execute, lines=123)
+
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_journalctl_output_with_units(self, mock_execute):
+        self._get_journalctl_output(mock_execute, units=['fake-unit1',
+                                                         'fake-unit2'])
+
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_journalctl_output_fail(self, mock_execute):
+        mock_execute.side_effect = processutils.ProcessExecutionError()
+        self.assertRaises(errors.CommandExecutionError,
+                          self._get_journalctl_output, mock_execute)
+
+    def test_gzip_and_b64encode(self):
+        contents = b'Squidward Tentacles'
+        io_dict = {'fake-name': io.BytesIO(bytes(contents))}
+        data = utils.gzip_and_b64encode(io_dict=io_dict)
+
+        res = io.BytesIO(base64.b64decode(data))
+        with tarfile.open(fileobj=res) as tar:
+            members = [(m.name, m.size) for m in tar]
+            self.assertEqual([('fake-name', len(contents))], members)
+
+            member = tar.extractfile('fake-name')
+            self.assertEqual(contents, member.read())
+
+    @mock.patch.object(utils, 'execute', autospec=True)
+    def test_get_command_output(self, mock_execute):
+        contents = b'Sandra Sandy Cheeks'
+        mock_execute.return_value = (contents, '')
+        data = utils.get_command_output(['foo'])
+
+        mock_execute.assert_called_once_with(
+            'foo', binary=True, log_stdout=False)
+        self.assertEqual(contents, data.read())
+
+    @mock.patch.object(subprocess, 'check_call')
+    def test_is_journalctl_present(self, mock_call):
+        self.assertTrue(utils.is_journalctl_present())
+
+    @mock.patch.object(subprocess, 'check_call')
+    def test_is_journalctl_present_false(self, mock_call):
+        os_error = OSError()
+        os_error.errno = errno.ENOENT
+        mock_call.side_effect = os_error
+        self.assertFalse(utils.is_journalctl_present())
+
+    @mock.patch.object(utils, 'gzip_and_b64encode')
+    @mock.patch.object(utils, 'is_journalctl_present')
+    @mock.patch.object(utils, 'get_command_output')
+    @mock.patch.object(utils, 'get_journalctl_output')
+    def test_collect_system_logs_journald(
+            self, mock_logs, mock_outputs, mock_journalctl, mock_gzip_b64):
+        mock_journalctl.return_value = True
+        ret = 'Patrick Star'
+        mock_gzip_b64.return_value = ret
+
+        logs_string = utils.collect_system_logs()
+        self.assertEqual(ret, logs_string)
+        mock_logs.assert_called_once_with(lines=None)
+        calls = [mock.call(['ps', '-ax']), mock.call(['df', '-a']),
+                 mock.call(['iptables', '-L']), mock.call(['ip', 'addr'])]
+        mock_outputs.assert_has_calls(calls, any_order=True)
+        mock_gzip_b64.assert_called_once_with(
+            file_list=[],
+            io_dict={'journal': mock.ANY, 'ip_addr': mock.ANY, 'ps': mock.ANY,
+                     'df': mock.ANY, 'iptables': mock.ANY})
+
+    @mock.patch.object(utils, 'gzip_and_b64encode')
+    @mock.patch.object(utils, 'is_journalctl_present')
+    @mock.patch.object(utils, 'get_command_output')
+    def test_collect_system_logs_non_journald(
+            self, mock_outputs, mock_journalctl, mock_gzip_b64):
+        mock_journalctl.return_value = False
+        ret = 'SpongeBob SquarePants'
+        mock_gzip_b64.return_value = ret
+
+        logs_string = utils.collect_system_logs()
+        self.assertEqual(ret, logs_string)
+        calls = [mock.call(['dmesg']), mock.call(['ps', '-ax']),
+                 mock.call(['df', '-a']), mock.call(['iptables', '-L']),
+                 mock.call(['ip', 'addr'])]
+        mock_outputs.assert_has_calls(calls, any_order=True)
+        mock_gzip_b64.assert_called_once_with(
+            file_list=['/var/log'],
+            io_dict={'iptables': mock.ANY, 'ip_addr': mock.ANY, 'ps': mock.ANY,
+                     'dmesg': mock.ANY, 'df': mock.ANY})