Add free-form text to qube for notes, comments, ...

qubesadmin part of adding free-form text to each qube for comments,
notes, descriptions, remarks, reminders, etc.

fixes: QubesOS/qubes-issues#899

Additional options to skip legacy backup tests or super slow tests

Author: alimirjamali

doc/conf.py

@@ -84,7 +84,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -344,6 +344,8 @@
         u'Kill the specified qube', _man_pages_author, 1),
     ('manpages/qvm-ls', 'qvm-ls',
         u'List VMs and various information about them', _man_pages_author, 1),
+    ('manpages/qvm-notes', 'qvm-notes',
+        u'Manipulate qube notes', _man_pages_author, 1),
     ('manpages/qvm-pause', 'qvm-pause',
         u'Pause a specified qube(s)', _man_pages_author, 1),
     ('manpages/qvm-pool', 'qvm-pool',
@@ -370,7 +372,6 @@
         u'Pause a qube', _man_pages_author, 1),
     ('manpages/qvm-volume', 'qvm-volume',
         u'Manage storage volumes of a qube', _man_pages_author, 1),
-
     ('manpages/qubes-prefs', 'qubes-prefs',
         u'Display system-wide Qubes settings', _man_pages_author, 1),
 ]

doc/manpages/qvm-notes.rst

@@ -0,0 +1,80 @@
+.. program:: qvm-notes
+
+:program:`qvm-notes` -- Manipulate qube notes
+=============================================
+
+Synopsis
+--------
+
+:command:`qvm-notes` [options] *VMNAME* [--edit | --print | --import *FILENAME* | --set '*NOTES*' | --append '*NOTES*' | --delete]
+
+Description
+-----------
+
+This command is used to manipulate individual qube notes. Each qube notes is
+limited to 256KB of clear text which could contain most UTF-8 characters.
+However, some UTF-8 characters will be replaced with underline (`_`) due to
+security limitations. Qube notes will be included in backup/restore.
+
+If this command is run outside dom0, it will require `admin.vm.notes.Get` and/or
+`admin.vm.notes.Set` access privileges for the target qube in the RPC policies.
+
+General options
+---------------
+
+.. option:: --verbose, -v
+
+   increase verbosity
+
+.. option:: --quiet, -q
+
+   decrease verbosity
+
+.. option:: --help, -h
+
+   show this help message and exit
+
+.. option:: --version
+
+   show program's version number and exit
+
+.. option:: --force, -f
+
+   Do not prompt for confirmation; assume `yes`
+
+Action options
+--------------
+
+.. option:: --edit, -e
+
+   Edit qube notes in $EDITOR (default text editor). This is the default action.
+
+.. option:: --print, -p
+
+   Print qube notes
+
+.. option:: --import=FILENAME, -i FILENAME
+
+   Import qube notes from file
+
+.. option:: --set='NOTES', -s 'NOTES'
+
+   Set qube notes from the provided string
+
+.. option:: --append='NOTES'
+
+   Append the provided string to qube notes
+
+.. option:: --delete, -d
+
+   Delete qube notes
+
+Authors
+-------
+
+| Marek Marczykowski <marmarek at invisiblethingslab dot com>
+| Ali Mirjamali <ali at mirjamali dot com>
+
+| For complete author list see: https://github.com/QubesOS/qubes-core-admin-client.git
+
+.. vim: ts=3 sw=3 et tw=80

qubesadmin/app.py

@@ -365,6 +365,7 @@ def clone_vm(self, src_vm, new_name, new_cls=None, *, pool=None, pools=None,
                  ignore_errors=False, ignore_volumes=None,
                  ignore_devices=False):
         # pylint: disable=too-many-statements
+        # pylint: disable=too-many-branches
         """Clone Virtual Machine
 
         Example usage with custom storage pools:
@@ -474,6 +475,16 @@ def clone_vm(self, src_vm, new_name, new_cls=None, *, pool=None, pools=None,
                     if not ignore_errors:
                         raise
 
+            try:
+                vm_notes = src_vm.get_notes()
+                if vm_notes:
+                    dst_vm.set_notes(vm_notes)
+            except qubesadmin.exc.QubesException as e:
+                dst_vm.log.error(
+                    'Failed to clone qube notes: {!s}'.format(e))
+                if not ignore_errors:
+                    raise
+
             try:
                 dst_vm.firewall.save_rules(src_vm.firewall.rules)
             except qubesadmin.exc.QubesException as e:

qubesadmin/backup/__init__.py

@@ -71,3 +71,7 @@ def included_in_backup(self):
     def handle_firewall_xml(self, vm, stream):
         '''Import appropriate format of firewall.xml'''
         raise NotImplementedError
+
+    def handle_notes_txt(self, vm, stream):
+        '''Import qubes notes.txt'''
+        raise NotImplementedError

qubesadmin/backup/core2.py

@@ -140,6 +140,10 @@ def _translate_action(key):
         except:  # pylint: disable=bare-except
             vm.log.exception('Failed to set firewall')
 
+    def handle_notes_txt(self, vm, stream):
+        '''Qube notes did not exist at this time'''
+        raise NotImplementedError
+
 
 class Core2Qubes(qubesadmin.backup.BackupApp):
     '''Parsed qubes.xml'''

qubesadmin/backup/core3.py

@@ -52,6 +52,13 @@ def handle_firewall_xml(self, vm, stream):
         except:  # pylint: disable=bare-except
             vm.log.exception('Failed to set firewall')
 
+    def handle_notes_txt(self, vm, stream):
+        '''Load new (Qubes >= 4.2) notes'''
+        try:
+            vm.set_notes(stream.read().decode())
+        except:  # pylint: disable=bare-except
+            vm.log.exception('Failed to set notes')
+
 class Core3Qubes(qubesadmin.backup.BackupApp):
     '''Parsed qubes.xml'''
     def __init__(self, store=None):

qubesadmin/backup/restore.py

@@ -1963,6 +1963,8 @@ def restore_do(self, restore_info):
                         handlers[img_path] = data_func
                     handlers[os.path.join(vm_info.subdir, 'firewall.xml')] = \
                         functools.partial(vm_info.vm.handle_firewall_xml, vm)
+                    handlers[os.path.join(vm_info.subdir, 'notes.txt')] = \
+                        functools.partial(vm_info.vm.handle_notes_txt, vm)
                     handlers[os.path.join(vm_info.subdir,
                         'whitelisted-appmenus.list')] = \
                         functools.partial(self._handle_appmenus_list, vm)

qubesadmin/exc.py

@@ -232,5 +232,9 @@ def __init__(self, prop):
         super().__init__("Failed to access '%s' property" % prop)
 
 
+
+class QubesNotesError(QubesException):
+    """Some problem with qube notes."""
+
 # legacy name
 QubesDaemonNoResponseError = QubesDaemonAccessError

qubesadmin/tests/app.py

@@ -358,6 +358,11 @@ def clone_setup_common_calls(self, src, dst):
             (dst, 'admin.vm.firewall.Set', None, rules)] = \
             b'0\x00'
 
+        # notes
+        self.app.expected_calls[
+            (src, 'admin.vm.notes.Get', None, None)] = \
+            b'0\0'
+
         # storage
         for vm in (src, dst):
             self.app.expected_calls[
@@ -813,6 +818,50 @@ def test_044_clone_devices_fail(self):
 
         self.assertAllCalled()
 
+    def test_045_clone_notes_fail(self):
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.notes.Get', None, None)] = \
+            b'0\0Secret Note\0'
+        self.app.expected_calls[
+            ('new-name', 'admin.vm.notes.Set', None, b'Secret Note\0')] = \
+            b'2\0QubesNotesException\0\0It was For Your Eyes Only!\0'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.property.List', None, None)] = \
+            b'0\0qid\nname\ntemplate\nlabel\nmemory\n'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.volume.List', None, None)] = \
+            b'0\x00'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.property.Get', 'label', None)] = \
+            b'0\0default=False type=label red'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.property.Get', 'template', None)] = \
+            b'0\0default=False type=vm test-template'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.property.Get', 'memory', None)] = \
+            b'0\0default=False type=int 400'
+        self.app.expected_calls[
+            ('new-name', 'admin.vm.property.Set', 'memory', b'400')] = \
+            b'0\0'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.tag.List', None, None)] = \
+            b'0\0'
+        self.app.expected_calls[
+            ('test-vm', 'admin.vm.feature.List', None, None)] = \
+            b'0\0'
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\x00new-name class=AppVM state=Halted\n' \
+            b'test-vm class=AppVM state=Halted\n' \
+            b'test-template class=TemplateVM state=Halted\n' \
+            b'test-net class=AppVM state=Halted\n'
+        self.app.expected_calls[('dom0', 'admin.vm.Create.AppVM',
+            'test-template', b'name=new-name label=red')] = b'0\x00'
+        self.app.expected_calls[('new-name', 'admin.vm.Remove', None, None)] = \
+            b'0\x00'
+        with self.assertRaises(qubesadmin.exc.QubesException):
+            self.app.clone_vm('test-vm', 'new-name')
+        self.assertAllCalled()
+
     def test_050_automatic_reset_cache(self):
         self.app.cache_enabled = True
         dispatcher = qubesadmin.events.EventsDispatcher(self.app)

qubesadmin/tests/backup/backupcompatibility.py

@@ -1050,6 +1050,14 @@ def create_v4_files(self):
                 "tests/backup/v4-firewall.xml"
             f_firewall.write(xml_path.read_bytes())
 
+        # setup notes only on one VM
+        with open(
+                self.fullpath("appvms/test-work/notes.txt"),
+                "w+",
+                encoding="utf-8",
+            ) as notes:
+            notes.write("For Your Eyes Only")
+
         # StandaloneVMs
         for vm in ('test-standalonevm', 'test-hvm'):
             os.mkdir(self.fullpath('appvms/{}'.format(vm)))
@@ -1528,6 +1536,8 @@ def create_limited_tmpdir(self, size):
         self.addCleanup(self.cleanup_tmpdir, tmpdir)
         return tmpdir.name
 
+    @unittest.skipIf(os.environ.get('DISABLE_LEGACY_TESTS', False),
+        'Set DISABLE_LEGACY_TESTS=1 environment variable to skip this test')
     def test_210_r2(self):
         self.create_v3_backup(False)
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
@@ -1601,6 +1611,8 @@ def test_210_r2(self):
 
         self.assertDom0Restored(dummy_timestamp)
 
+    @unittest.skipIf(os.environ.get('DISABLE_LEGACY_TESTS', False),
+        'Set DISABLE_LEGACY_TESTS=1 environment variable to skip this test')
     def test_220_r2_encrypted(self):
         self.create_v3_backup(True)
 
@@ -1676,6 +1688,8 @@ def test_220_r2_encrypted(self):
 
         self.assertDom0Restored(dummy_timestamp)
 
+    @unittest.skipIf(os.environ.get('DISABLE_LEGACY_TESTS', False),
+        'Set DISABLE_LEGACY_TESTS=1 environment variable to skip this test')
     def test_230_r2_uncompressed(self):
         self.create_v3_backup(False, False)
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
@@ -1781,6 +1795,9 @@ def test_230_r4(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()
 
@@ -1858,6 +1875,9 @@ def test_230_r4_compressed(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()
 
@@ -1935,6 +1955,9 @@ def test_230_r4_custom_cmpression(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()
 
@@ -2049,6 +2072,9 @@ def test_230_r4_uncommon_compression_forced(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()
 
@@ -2126,6 +2152,9 @@ def test_230_r4_optional_compression(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()
 
@@ -2171,6 +2200,8 @@ def test_230_r4_optional_compression(self):
 
         self.assertDom0Restored(dummy_timestamp)
 
+    @unittest.skipIf(os.environ.get('DISABLE_SUPER_SLOW_TESTS', False),
+        'Set DISABLE_SUPER_SLOW_TESTS=1 environment variable to skip this test')
     @unittest.skipUnless(shutil.which('scrypt'),
         "scrypt not installed")
     def test_300_r4_no_space(self):
@@ -2202,6 +2233,9 @@ def test_300_r4_no_space(self):
         self.app.expected_calls[
             ('test-work', 'admin.vm.firewall.Set', None,
             firewall_data.encode())] = b'0\0'
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.notes.Set', None,
+            b'For Your Eyes Only')] = b'0\0'
 
         qubesd_calls_queue = multiprocessing.Queue()