#!/usr/bin/python3

# This file is part of Cockpit.
#
# Copyright (C) 2021 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

import os
import sys

# import Cockpit's machinery for test VMs and its browser test API
TEST_DIR = os.path.dirname(__file__)
sys.path.append(os.path.join(TEST_DIR, "common"))
sys.path.append(os.path.join(os.path.dirname(TEST_DIR), "bots/machine"))

from machineslib import VirtualMachinesCase  # noqa
from testlib import no_retry_when_changed, nondestructive, test_main, wait, Error, timeout  # noqa


def get_next_free_target(used_targets):
    i = 0
    while ("vd" + chr(97 + i) in used_targets):
        i += 1

    used_targets.append("vd" + chr(97 + i))
    return used_targets


def release_target(used_targets, target):
    used_targets.remove(target)


@nondestructive
class TestMachinesDisks(VirtualMachinesCase):

    def wait_for_disk_stats(self, name, target):
        b = self.browser
        try:
            with b.wait_timeout(10):
                b.wait_visible("#vm-{0}-disks-{1}-used".format(name, target))  # wait for disk statistics to show up
        except Error as ex:
            if not ex.msg.startswith('timeout'):
                raise
            # stats did not show up, check if user message showed up
            print("Libvirt version does not support disk statistics")
            b.wait_visible("#vm-{0}-disks-notification".format(name))

    def testDiskEdit(self):
        b = self.browser
        m = self.machine

        def open(target):
            b.click("#vm-subVmTest1-disks-{0}-edit".format(target))
            b.wait_visible("#vm-subVmTest1-disks-{0}-edit-dialog".format(target))

        def cancel(target):
            b.click("#vm-subVmTest1-disks-{0}-edit-dialog-cancel".format(target))
            b.wait_not_present("#vm-subVmTest1-disks-{0}-edit-dialog".format(target))

        def save(target, xfail=None):
            b.click("#vm-subVmTest1-disks-{0}-edit-dialog-save".format(target))
            if xfail:
                b.wait_in_text("#vm-subVmTest1-disks-{0}-edit-dialog .pf-c-alert".format(target), xfail)
            else:
                b.wait_not_present("#vm-subVmTest1-disks-{0}-edit-dialog".format(target))

        self.createVm("subVmTest1")

        # prepare libvirt storage pools
        p1 = os.path.join(self.vm_tmpdir, "vm_one")
        m.execute("mkdir --mode 777 {0}".format(p1))
        m.execute("virsh pool-create-as myPoolOne --type dir --target {0}".format(p1))
        m.execute("virsh vol-create-as myPoolOne mydisk --capacity 100M --format raw")  # raw support shareable
        m.execute("virsh vol-create-as myPoolOne mydisk2 --capacity 100M --format raw")
        m.execute("virsh vol-create-as myPoolOne mydisk3 --capacity 100M --format qcow2")
        m.execute("virsh vol-create-as myPoolOne mydisk4 --capacity 100M --format qcow2")
        wait(lambda: all(disk in m.execute("virsh vol-list myPoolOne") for disk in ["mydisk", "mydisk2", "mydisk3", "mydisk4"]))

        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydisk --target vde --targetbus virtio --persistent".format(p1))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydisk2 --target vdf --targetbus virtio".format(p1))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydisk3 --target vdg --targetbus virtio --subdriver qcow2 --persistent".format(p1))

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")
        self.waitVmRow("subVmTest1")
        b.wait_in_text("#vm-subVmTest1-state", "Running")

        self.goToVmPage("subVmTest1")

        # Test non-persistent disks are not configurable
        b.wait_not_present("#vm-subVmTest1-disks-vdf-edit")

        # Test close button
        open("vdg")
        b.click("#vm-subVmTest1-disks-vdg-edit-dialog > button:first-child")
        b.wait_not_present("#vm-subVmTest1-disks-vdg-edit-dialog")

        # Test qcow2 disk has only readonly attribute configurable
        open("vdg")
        b.wait_visible("#vm-subVmTest1-disks-vdg-edit-readonly")
        b.wait_visible("#vm-subVmTest1-disks-vdg-edit-writable")
        b.wait_not_present("#vm-subVmTest1-disks-vdg-edit-writable-shareable")
        cancel("vdg")

        # Test configuration of readonly and shareable attributes
        open("vde")
        # Changing readonly with running VM
        b.set_checked("#vm-subVmTest1-disks-vde-edit-readonly", True)

        # Tooltip in dialog should show
        b.wait_visible("#vm-subVmTest1-disks-vde-edit-idle-message")

        # Save changes
        save("vde")
        # See tooltip present in disk listing table
        b.wait_visible("#vm-subVmTest1-disks-vde-access-tooltip")

        # Shut off VM and see state has changed
        self.performAction("subVmTest1", "forceOff")
        # Check change has been applied after shutoff
        b.wait_in_text("#vm-subVmTest1-disks-vde-access", "Read-only")
        # See tooltip no longer present in disk listing table
        b.wait_not_present("#vm-subVmTest1-disks-vde-access-tooltip")

        # Test configuration of readonly and shareable attributes for Shut off VM
        open("vde")
        # Changing readonly
        b.set_checked("#vm-subVmTest1-disks-vde-edit-writable-shareable", True)
        # Tooltip in dialog should not be present
        b.wait_not_present("#vm-subVmTest1-disks-vde-edit-idle-message")

        # Close dialog
        save("vde")
        b.wait_in_text("#vm-subVmTest1-disks-vde-access", "Writeable and shared")
        b.wait_not_present("#vm-subVmTest1-disks-vde-access-tooltip")

        b.wait_in_text("#vm-subVmTest1-disks-vde-bus", "virtio")
        b.wait_not_present("#vm-subVmTest1-disks-vde-cache")
        # Change bus type to scsi and cache mode to writeback
        open("vde")
        b.select_from_dropdown("#vm-subVmTest1-disks-vde-edit-bus-type", "scsi")
        b.select_from_dropdown("#vm-subVmTest1-disks-vde-edit-cache-mode", "writeback")

        # Close dialog
        save("vde")
        # Target has changed from vdX to sdX
        b.wait_in_text("#vm-subVmTest1-disks-sda-bus", "scsi")
        b.wait_in_text("#vm-subVmTest1-disks-sda-cache", "writeback")

        if m.execute("virsh --version") >= "6.10.0":
            # Check that errors appear correctly and dissappear when closing the dialog and reopening
            m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydisk4 --target sdb --targetbus sata --persistent".format(p1))
            b.reload()
            b.enter_page('/machines')
            open("sdb")
            b.set_checked("#vm-subVmTest1-disks-sdb-edit-readonly", True)
            save("sdb", "readonly sata disks are not supported")
            cancel("sdb")
            open("sdb")
            b.wait_not_present("#vm-subVmTest1-disks-sdb-edit-dialog .pf-c-alert")
            cancel("sdb")

        # Start Vm
        b.click("#vm-subVmTest1-run")
        b.wait_in_text("#vm-subVmTest1-state", "Running")

        # Test disk's bus and cache cannot be changed on running VM
        open("sda")
        b.wait_visible("#vm-subVmTest1-disks-sda-edit-bus-type:disabled")
        b.wait_visible("#vm-subVmTest1-disks-sda-edit-cache-mode:disabled")

        # Disks on non-persistent VM cannot be edited
        m.execute("virsh undefine subVmTest1")
        b.wait_not_present("#vm-subVmTest1-disks-vde-edit")

    def testDisks(self):
        b = self.browser
        m = self.machine

        self.createVm("subVmTest1")

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")
        self.waitVmRow("subVmTest1")

        b.wait_in_text("#vm-subVmTest1-state", "Running")

        self.goToVmPage("subVmTest1")

        # Test basic disk properties
        b.wait_in_text("#vm-subVmTest1-disks-vda-bus", "virtio")

        b.wait_in_text("#vm-subVmTest1-disks-vda-device", "disk")

        b.wait_in_text("#vm-subVmTest1-disks-vda-source-file", "/var/lib/libvirt/images/subVmTest1-2.img")

        # Test domstats
        self.wait_for_disk_stats("subVmTest1", "vda")
        if b.is_present("#vm-subVmTest1-disks-vda-used"):
            b.wait_in_text("#vm-subVmTest1-disks-vda-used", "0.0")

        # Test add disk by external action
        m.execute("qemu-img create -f raw /var/lib/libvirt/images/image3.img 128M")
        # attach to the virtio bus instead of ide
        m.execute("virsh attach-disk subVmTest1 /var/lib/libvirt/images/image3.img vdc")

        b.wait_visible("#vm-subVmTest1-disks-vda-used")

        b.wait_in_text("#vm-subVmTest1-disks-vda-bus", "virtio")

        b.wait_in_text("#vm-subVmTest1-disks-vdc-bus", "virtio")
        b.wait_in_text("#vm-subVmTest1-disks-vdc-device", "disk")
        b.wait_in_text("#vm-subVmTest1-disks-vdc-source-file", "/var/lib/libvirt/images/image3.img")

        self.wait_for_disk_stats("subVmTest1", "vdc")
        if b.is_present("#vm-subVmTest1-disks-vdc-used"):
            b.wait_in_text("#vm-subVmTest1-disks-vdc-used", "0")
            b.wait_in_text("#vm-subVmTest1-disks-vdc-capacity", "0.12")  # 128 MB

        # Test remove disk - by external action
        m.execute("virsh detach-disk subVmTest1 vdc")
        print("Restarting vm-subVmTest1, might take a while")
        self.performAction("subVmTest1", "forceReboot")

        b.wait_visible("#vm-subVmTest1-disks-vda-device")
        b.wait_not_present("#vm-subVmTest1-disks-vdc-device")

        # Check when no storage pool and adding existed disk
        # delete the default pool
        m.execute("virsh pool-destroy images && virsh pool-undefine images")
        # Open "add disk" dialog
        b.click("#vm-subVmTest1-disks-adddisk")
        b.click("#vm-subVmTest1-disks-adddisk-useexisting")
        # Check
        b.wait_not_present("#navbar-oops")
        b.wait_visible("#vm-subVmTest1-disks-adddisk-existing-select-pool:disabled")
        b.wait_in_text("#vm-subVmTest1-disks-adddisk-existing-select-pool:disabled", "No storage pools available")
        b.wait_visible("#vm-subVmTest1-disks-adddisk-dialog-add:disabled")

    class VMAddDiskDialog(object):
        def __init__(
            self, test_obj, pool_name=None, volume_name=None,
            vm_name='subVmTest1',
            file_path=None, device=None,
            volume_size=10, volume_size_unit='MiB',
            mode="create-new",
            expected_target='vda', permanent=False, cache_mode=None,
            bus_type=None, verify=True, pool_type=None,
            volume_format=None,
            persistent_vm=True,
            pixel_test_tag=None
        ):
            self.test_obj = test_obj
            self.vm_name = vm_name
            self.file_path = file_path
            self.device = device
            self.pool_name = pool_name
            self.mode = mode
            self.volume_name = volume_name
            self.volume_size = volume_size
            self.volume_size_unit = volume_size_unit
            self.expected_target = expected_target
            self.permanent = permanent
            self.cache_mode = cache_mode
            self.bus_type = bus_type
            self.verify = verify
            self.pool_type = pool_type
            self.volume_format = volume_format
            self.persistent_vm = persistent_vm

            self.pixel_test_tag = pixel_test_tag

        def getExpectedFormat(selfi, pool_type):
            # Guess by the name of the pool it's format to avoid passing more parameters
            if pool_type == 'iscsi':
                return 'unknown'
            elif pool_type == 'disk':
                return 'none'
            else:
                return 'qcow2'

        def execute(self):
            self.open()
            self.fill()

            if self.pixel_test_tag:
                self.test_obj.browser.assert_pixels("#vm-subVmTest1-disks-adddisk-dialog-modal-window", self.pixel_test_tag,
                                                    ignore=[".pf-c-expandable-section__toggle-icon"])

            if self.verify:
                self.add_disk()
                self.verify_disk_added()

        def open(self):
            b = self.test_obj.browser
            b.click("#vm-{0}-disks-adddisk".format(self.vm_name))  # button
            b.wait_in_text(".pf-c-modal-box__title", "Add disk")

            b.wait_visible("label:contains(Create new)")
            if self.mode == "use-existing":
                b.click("label:contains(Use existing)")
            elif self.mode == "custom-path":
                b.click("label:contains(Custom path)")

            return self

        def fill(self):
            b = self.test_obj.browser
            if self.mode == "create-new":
                # Choose storage pool
                if not self.pool_type or self.pool_type not in ['iscsi', 'iscsi-direct']:
                    b.wait_visible("#vm-{0}-disks-adddisk-new-select-pool:enabled".format(self.vm_name))
                    b.select_from_dropdown("#vm-{0}-disks-adddisk-new-select-pool".format(self.vm_name), self.pool_name)
                else:
                    b.click("#vm-{0}-disks-adddisk-new-select-pool".format(self.vm_name))
                    # Our custom select does not respond on the click function
                    b._wait_present(".pf-c-modal-box option[value={0}]:disabled".format(self.pool_name))
                    return self

                # Insert name for the new volume
                b.set_input_text("#vm-{0}-disks-adddisk-new-name".format(self.vm_name), self.volume_name)
                # Insert size for the new volume
                b.set_input_text("#vm-{0}-disks-adddisk-new-size".format(self.vm_name), str(self.volume_size))
                b.select_from_dropdown("#vm-{0}-disks-adddisk-new-unit".format(self.vm_name), self.volume_size_unit)

                if self.volume_format:
                    b.select_from_dropdown("#vm-{0}-disks-adddisk-new-format".format(self.vm_name), self.volume_format)
                else:
                    b.wait_val("#vm-{0}-disks-adddisk-new-format".format(self.vm_name), self.getExpectedFormat(self.pool_type))
            elif self.mode == "custom-path":
                b.wait_visible("#vm-{0}-disks-adddisk-file".format(self.vm_name))
                # Type in file path
                b.set_file_autocomplete_val("#vm-{0}-disks-adddisk-file".format(self.vm_name), self.file_path)
                if self.device:
                    b.select_from_dropdown("#vm-{0}-disks-adddisk-select-device".format(self.vm_name), self.device)
            elif self.mode == "use-existing":
                b.wait_visible("#vm-{0}-disks-adddisk-existing-select-pool:enabled".format(self.vm_name))
                # Choose storage pool
                b.select_from_dropdown("#vm-{0}-disks-adddisk-existing-select-pool".format(self.vm_name), self.pool_name)
                # Select from the available volumes
                b.select_from_dropdown("#vm-{0}-disks-adddisk-existing-select-volume".format(self.vm_name), self.volume_name)

            # Configure persistency - by default the check box in unchecked for running VMs
            if self.permanent:
                b.click("#vm-{0}-disks-adddisk-permanent".format(self.vm_name))

            # Check non-persistent VM cannot have permanent disk attached
            if not self.persistent_vm:
                b.wait_not_present("#vm-{0}-disks-adddisk-new-permanent".format(self.vm_name))

            # Configure performance options
            if self.cache_mode and b.is_visible("div.pf-c-modal-box button:contains(Show additional options)"):
                b.click("div.pf-c-modal-box button:contains(Show additional options)")
                b.select_from_dropdown("#cache-mode", self.cache_mode)
            else:
                b.wait_not_visible("#cache-mode")

            # Configure bus type
            if self.bus_type and b.is_visible("div.pf-c-modal-box button:contains(Show additional options)"):
                b.click("div.pf-c-modal-box button:contains(Show additional options)")
                b.select_from_dropdown("div.pf-c-modal-box #bus-type", self.bus_type)

            return self

        def add_disk(self):
            b = self.test_obj.browser
            b.click(".pf-c-modal-box__footer button:contains(Add)")
            b.wait_not_present("#vm-{0}-disks-adddisk-dialog-modal-window".format(self.vm_name))

            return self

        def verify_disk_added(self):
            b = self.test_obj.browser
            m = self.test_obj.machine
            if self.device == "cdrom" or (self.file_path and self.file_path.endswith(".iso")):
                expected_bus_type = self.bus_type or "scsi"
                expected_device = self.device or "cdrom"
            else:
                expected_bus_type = self.bus_type or "virtio"
                expected_device = self.device or "disk"

            b.wait_in_text("#vm-{0}-disks-{1}-bus".format(self.vm_name, self.expected_target), expected_bus_type)
            b.wait_in_text("#vm-{0}-disks-{1}-device".format(self.vm_name, self.expected_target), expected_device)

            # Check volume was added to pool's volume list
            if self.mode == "create-new":
                self.test_obj.goToMainPage()
                b.click(".pf-c-card .pf-c-card__header button:contains(Storage pool)")

                self.test_obj.waitPoolRow(self.pool_name)
                self.test_obj.togglePoolRow(self.pool_name)

                b.wait_visible("#pool-{0}-system-storage-volumes".format(self.pool_name))
                b.click("#pool-{0}-system-storage-volumes".format(self.pool_name))  # open the "Storage volumes" subtab
                b.wait_visible("#pool-{0}-system-volume-{1}-name".format(self.pool_name, self.volume_name))

                b.click(".machines-listing-breadcrumb li a:contains(Virtual machines)")
                self.test_obj.goToVmPage("subVmTest1")

            # Detect volume format
            if not self.mode == "custom-path":
                volume_xml = m.execute("virsh vol-dumpxml {0} {1} ".format(self.volume_name, self.pool_name))
                detect_format_cmd = "echo \"{0}\" | xmllint --xpath '{1}' -"

                b.wait_in_text('#vm-{0}-disks-{1}-source-volume'.format(self.vm_name, self.expected_target), self.volume_name)

                expected_format = self.getExpectedFormat(self.pool_type)
                # Unknown pool format isn't present in xml anymore
                if expected_format == "unknown" and m.execute("virsh --version") >= "5.6.0":
                    m.execute(detect_format_cmd.format(volume_xml, "/volume/target") + " | grep -qv format")
                else:
                    vol_xml = m.execute(detect_format_cmd.format(volume_xml, "/volume/target/format")).rstrip()
                    self.test_obj.assertEqual(vol_xml, '<format type="{0}"/>'.format(self.volume_format or expected_format))
            else:
                b.wait_in_text('#vm-{0}-disks-{1}-source-file'.format(self.vm_name, self.expected_target), self.file_path)

            if self.cache_mode:
                b.wait_in_text("#vm-{0}-disks-{1}-cache".format(self.vm_name, self.expected_target), self.cache_mode)

            return self

    def testAddDiskSCSI(self):
        b = self.browser
        m = self.machine

        used_targets = ['vda']

        # Prepare an iscsi pool
        # Debian images' -cloud kernel don't have target-cli-mod kmod
        if "debian" not in m.image:
            # Preparations for testing ISCSI pools

            target_iqn = "iqn.2019-09.cockpit.lan"
            self.prepareStorageDeviceOnISCSI(target_iqn)

            m.execute("virsh pool-define-as iscsi-pool --type iscsi --target /dev/disk/by-id --source-host 127.0.0.1 --source-dev {0} && virsh pool-start iscsi-pool".format(target_iqn))
            wait(lambda: "unit:0:0:0" in self.machine.execute("virsh pool-refresh iscsi-pool && virsh vol-list iscsi-pool"), delay=3)

            self.addCleanup(self.machine.execute, "virsh pool-destroy iscsi-pool; virsh pool-delete iscsi-pool; virsh pool-undefine iscsi-pool")

        args = self.createVm("subVmTest1")

        # Wait for the system to completely start
        wait(lambda: "login as 'cirros' user." in self.machine.execute("cat {0}".format(args["logfile"])), delay=3)

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        if "debian" not in m.image and "ubuntu" not in m.image:
            # ISCSI driver does not support virStorageVolCreate API
            self.VMAddDiskDialog(
                self,
                pool_name='iscsi-pool',
                pool_type='iscsi',
                verify=False,
            ).execute()

            self.VMAddDiskDialog(
                self,
                pool_name='iscsi-pool',
                pool_type='iscsi',
                volume_name='unit:0:0:0',
                expected_target=get_next_free_target(used_targets)[-1],
                mode='use-existing',
            ).execute()

            # Detach the iscsi disk before reaching teardown because shutting of the domains would sometimes hang when iscsi disks are attached
            m.execute("virsh detach-disk subVmTest1 --target vdb")

        # AppArmor doesn't like the non-standard path for our storage pools
        if m.image in ["debian-testing"]:
            self.allow_journal_messages('.* type=1400 .* apparmor="DENIED" operation="open" profile="libvirt.* name="%s.*' % self.vm_tmpdir)

    @no_retry_when_changed
    def testAddDiskNFS(self):
        b = self.browser
        m = self.machine

        used_targets = ['vda']

        m.execute("if selinuxenabled 2>/dev/null; then setsebool -P virt_use_nfs 1; fi")

        # Prepare a local NFS pool
        self.restore_file("/etc/exports")
        nfs_pool = os.path.join(self.vm_tmpdir, "nfs_pool")
        mnt_exports = os.path.join(self.vm_tmpdir, "mnt_exports")
        m.execute("mkdir {0} {1} && echo '{1} 127.0.0.1/24(rw,sync,no_root_squash,no_subtree_check,fsid=0)' > /etc/exports".format(nfs_pool, mnt_exports))
        m.execute("systemctl restart nfs-server")
        m.execute("virsh pool-define-as nfs-pool --type netfs --target {0} --source-host 127.0.0.1 --source-path {1} && virsh pool-start nfs-pool".format(nfs_pool, mnt_exports))
        # And create a volume on it in order to test use existing volume dialog
        m.execute("virsh vol-create-as --pool nfs-pool --name nfs-volume-0 --capacity 1M --format qcow2")

        args = self.createVm("subVmTest1")

        # Wait for the system to completely start
        wait(lambda: "login as 'cirros' user." in self.machine.execute("cat {0}".format(args["logfile"])), delay=3)

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        self.VMAddDiskDialog(
            self,
            pool_name='nfs-pool',
            volume_name='nfs-volume-0',
            mode='use-existing',
            volume_size=1,
            volume_size_unit='MiB',
            expected_target=get_next_free_target(used_targets)[-1],
            pixel_test_tag='vm-add-disk-modal-nfs'
        ).execute()

        self.VMAddDiskDialog(
            self,
            pool_name='nfs-pool',
            volume_name='nfs-volume-1',
            volume_size=1,
            volume_size_unit='MiB',
            expected_target=get_next_free_target(used_targets)[-1],
        ).execute()

        # AppArmor doesn't like the non-standard path for our storage pools
        if m.image in ["debian-testing"]:
            self.allow_journal_messages('.* type=1400 .* apparmor="DENIED" operation="open" profile="libvirt.* name="%s.*' % self.vm_tmpdir)

    def testAddDiskPool(self):
        b = self.browser
        m = self.machine

        dev = self.add_ram_disk(2)
        used_targets = ['vda']

        args = self.createVm("subVmTest1")

        # Wait for the system to completely start
        wait(lambda: "login as 'cirros' user." in self.machine.execute("cat {0}".format(args["logfile"])), delay=3)

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        cmds = [
            "virsh pool-define-as pool-disk disk - - {0} - {1}".format(dev, os.path.join(self.vm_tmpdir, "poolDiskImages")),
            "virsh pool-build pool-disk --overwrite",
            "virsh pool-start pool-disk",
        ]
        self.machine.execute(" && ".join(cmds))
        partition = os.path.basename(dev) + "1"
        self.VMAddDiskDialog(
            self,
            pool_name='pool-disk',
            pool_type='disk',
            volume_name=partition,
            volume_size=1,
            volume_size_unit='MiB',
            expected_target=get_next_free_target(used_targets)[-1],
            pixel_test_tag='vm-add-disk-modal-disk-pool'
        ).execute()

        loop_dev = self.add_loop_disk()
        existing_disk = os.path.basename(loop_dev) + "p1"
        cmds = [
            "virsh pool-define-as {0} disk - - {1} - {2}".format("loop-disk", loop_dev, os.path.join(self.vm_tmpdir, "poolLoopImages")),
            "virsh pool-build {} --overwrite".format("loop-disk"),
            "virsh pool-start {}".format("loop-disk"),
            "virsh vol-create-as {} {} 1M".format("loop-disk", existing_disk)
        ]
        self.machine.execute(" && ".join(cmds))
        self.VMAddDiskDialog(
            self,
            mode="use-existing",
            pool_name="loop-disk",
            pool_type="disk",
            volume_name=existing_disk,
            expected_target=get_next_free_target(used_targets)[-1],
        ).execute()

        # AppArmor doesn't like the non-standard path for our storage pools
        if m.image in ["debian-testing"]:
            self.allow_journal_messages('.* type=1400 .* apparmor="DENIED" operation="open" profile="libvirt.* name="%s.*' % self.vm_tmpdir)

    @timeout(900)
    @no_retry_when_changed
    def testAddDiskDirPool(self):
        b = self.browser
        m = self.machine

        used_targets = ['vda']
        transient_targets = []

        # prepare libvirt storage pools
        v1 = os.path.join(self.vm_tmpdir, "vm_one")
        v2 = os.path.join(self.vm_tmpdir, "vm_two")
        default_tmp = os.path.join(self.vm_tmpdir, "default_tmp")
        m.execute("mkdir --mode 777 {0} {1} {2}".format(v1, v2, default_tmp))
        m.execute("virsh pool-define-as default_tmp --type dir --target {0} && virsh pool-start default_tmp".format(default_tmp))
        m.execute("virsh pool-define-as myPoolOne --type dir --target {0} && virsh pool-start myPoolOne".format(v1))
        m.execute("virsh pool-define-as myPoolTwo --type dir --target {0} && virsh pool-start myPoolTwo".format(v2))

        m.execute("virsh vol-create-as default_tmp defaultVol --capacity 10M --format raw")
        m.execute("virsh vol-create-as myPoolTwo mydiskofpooltwo_temporary --capacity 50M --format qcow2")
        m.execute("virsh vol-create-as myPoolTwo mydiskofpooltwo_permanent --capacity 50M --format qcow2")
        wait(lambda: "mydiskofpooltwo_permanent" in m.execute("virsh vol-list myPoolTwo"))

        self.createVm("subVmTest1")

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        transient_targets.append(get_next_free_target(used_targets)[-1])
        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            volume_name='mydiskofpoolone_temporary',
            mode='create-new',
            volume_size=10,
            volume_size_unit='MiB',
            permanent=False,
            expected_target=transient_targets[-1],
        ).execute()

        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            mode='use-existing',
        ).open()
        b.select_from_dropdown("#vm-subVmTest1-disks-adddisk-existing-select-pool", "myPoolOne")
        # since both disks are already attached
        b.wait_attr("#vm-subVmTest1-disks-adddisk-existing-select-volume", "disabled", "")
        b.wait_in_text("#vm-subVmTest1-disks-adddisk-existing-select-volume", "The pool is empty")
        b.click("#vm-subVmTest1-disks-adddisk-dialog-cancel")
        b.wait_not_present("#vm-subVmTest1-test-disks-adddisk-dialog-modal-window")

        self.VMAddDiskDialog(
            self,
            pool_name='myPoolTwo',
            volume_name='mydiskofpooltwo_permanent',
            volume_size=2,
            permanent=True,
            mode='use-existing',
            expected_target=get_next_free_target(used_targets)[-1],
        ).execute()

        # check the autoselected options
        # default_tmp pool should be autoselected since it's the first in alphabetical order
        # defaultVol volume should be autoselected since it's the only volume in default_tmp pool
        transient_targets.append(get_next_free_target(used_targets)[-1])
        self.VMAddDiskDialog(
            self,
            pool_name='default_tmp',
            volume_name='defaultVol',
            mode='use-existing',
            expected_target=transient_targets[-1],
            volume_format='raw',
        ).open().add_disk().verify_disk_added()

        # shut off
        self.performAction("subVmTest1", "forceOff")

        # check if the just added non-permanent disks are gone
        for target in transient_targets:
            b.wait_not_present("#vm-subVmTest1-disks-{0}-device".format(target))
            release_target(used_targets, target)

        permanent_targets = [t for t in used_targets if t not in transient_targets]
        for target in permanent_targets:
            b.wait_visible("#vm-subVmTest1-disks-{0}-device".format(target))

        # Apparmor on debian and ubuntu may prevent access to /dev/sdb1 when starting VM,
        # https://bugs.launchpad.net/ubuntu/+source/libvirt/+bug/1677398
        if "debian" not in m.image and "ubuntu" not in m.image:
            # Run VM
            b.click("#vm-subVmTest1-run")
            b.wait_in_text("#vm-subVmTest1-state", "Running")
            # Test disk attachment to non-persistent VM
            m.execute("virsh undefine subVmTest1")
            self.VMAddDiskDialog(
                self,
                pool_name='myPoolOne',
                volume_name='non-peristent-vm-disk',
                permanent=False,
                persistent_vm=False,
                expected_target=get_next_free_target(used_targets)[-1],
            ).execute()

        self.goToMainPage()

        # Undefine all Storage pools and  confirm that the Add Disk dialog is disabled
        active_pools = filter(lambda pool: pool != '', m.execute("virsh pool-list --name").split('\n'))
        for pool in active_pools:
            m.execute("virsh pool-destroy {0}".format(pool))
        b.wait_in_text("#card-pf-storage-pools .active-resources:nth-of-type(1)", "0")
        inactive_pools = filter(lambda pool: pool != '', m.execute("virsh pool-list --inactive --name").split('\n'))
        for pool in inactive_pools:
            m.execute("virsh pool-undefine {0}".format(pool))
        b.wait_in_text("#card-pf-storage-pools .active-resources:nth-of-type(2)", "0")

        self.goToVmPage('subVmTest1')
        b.click("#vm-subVmTest1-disks-adddisk")  # radio button label in modal dialog
        b.wait_visible("#vm-subVmTest1-disks-adddisk-dialog-add:disabled")
        b.click("label:contains(Use existing)")
        b.wait_visible("#vm-subVmTest1-disks-adddisk-dialog-add:disabled")
        b.click(".pf-c-modal-box__footer button:contains(Cancel)")

        # Make sure that trying to inspect the Disks tab will just show the fields that are available when a pool is inactive
        b.reload()
        b.enter_page('/machines')
        b.wait_in_text("body", "Virtual machines")
        # Check that usage information can't be fetched since the pool is inactive
        b.wait_not_present("#vm-subVmTest1-disks-vdd-used")

        # AppArmor doesn't like the non-standard path for our storage pools
        if m.image in ["debian-testing"]:
            self.allow_journal_messages('.* type=1400 .* apparmor="DENIED" operation="open" profile="libvirt.* name="%s.*' % self.vm_tmpdir)

    def testAddDiskCustomPath(self):
        b = self.browser
        m = self.machine

        self.createVm("subVmTest1")

        # Prepare file for Custom File disk type
        m.execute("touch /var/lib/libvirt/novell.iso")
        m.execute("touch /var/lib/libvirt/novell.qcow2")

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        # check disk device type
        self.VMAddDiskDialog(
            self,
            device='disk',
            bus_type='scsi',
            expected_target='sda',
            file_path='/var/lib/libvirt/novell.iso',
            mode='custom-path',
            pixel_test_tag='vm-add-disk-modal-custom-path'
        ).execute()

        # non iso file
        self.VMAddDiskDialog(
            self,
            device='disk',
            bus_type='scsi',
            expected_target='sdb',
            file_path='/var/lib/libvirt/novell.qcow2',
            mode='custom-path'
        ).execute()

        # shut off
        self.performAction("subVmTest1", "forceOff")

        # check cdrom device (cdrom can be only added to shut off VM)
        self.VMAddDiskDialog(
            self,
            device='cdrom',
            bus_type='scsi',
            expected_target='sda',
            file_path='/var/lib/libvirt/novell.iso',
            mode='custom-path'
        ).execute()

        # check that bus and device type can be automatically picked up from the *.iso extension
        # https://bugzilla.redhat.com/show_bug.cgi?id=1977810
        self.VMAddDiskDialog(
            self,
            expected_target='sda',
            file_path='/var/lib/libvirt/novell.iso',
            mode='custom-path'
        ).execute()

    @no_retry_when_changed
    def testAddDiskAdditionalOptions(self):
        b = self.browser
        m = self.machine

        used_targets = ['vda']

        self.createVm("subVmTest1", running=False)

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")

        self.goToVmPage("subVmTest1")

        # prepare libvirt storage pools
        v1 = os.path.join(self.vm_tmpdir, "vm_one")
        m.execute("mkdir --mode 777 {0}".format(v1))
        m.execute("virsh pool-define-as myPoolOne --type dir --target {0} && virsh pool-start myPoolOne".format(v1))

        # Configure cache mode from the UI
        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            volume_name='writeback_cache_disk',
            mode='create-new',
            volume_size=2,
            cache_mode='writeback',
            expected_target=get_next_free_target(used_targets)[-1],
            pixel_test_tag='vm-add-disk-modal-additional-options'
        ).execute()

        # Configure scsi bus type from the UI
        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            volume_name='scsi_bus_disk',
            mode='create-new',
            bus_type='scsi',
            expected_target='sda',
        ).execute()

        # Configure usb bus type from the UI
        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            volume_name='usb_bus_disk',
            mode='create-new',
            bus_type='usb',
            expected_target='sdb',
        ).execute()

        # testing sata disk after VM shutoff because sata disk cannot be hotplugged
        self.VMAddDiskDialog(
            self,
            pool_name='myPoolOne',
            volume_name='sata_bus_disk',
            mode='create-new',
            bus_type='sata',
            expected_target='sdc',
        ).execute()

    def testDetachDisk(self):
        b = self.browser
        m = self.machine

        # prepare libvirt storage pools
        p1 = os.path.join(self.vm_tmpdir, "vm_one")
        m.execute("mkdir --mode 777 {0}".format(p1))
        m.execute("virsh pool-create-as myPoolOne --type dir --target {0}".format(p1))
        m.execute("virsh vol-create-as myPoolOne mydiskofpoolone_1 --capacity 1G --format qcow2")
        m.execute("virsh vol-create-as myPoolOne mydiskofpoolone_2 --capacity 1G --format qcow2")
        m.execute("virsh vol-create-as myPoolOne mydiskofpoolone_3 --capacity 1M --format qcow2")
        wait(lambda: "mydiskofpoolone_1" in m.execute("virsh vol-list myPoolOne"))
        wait(lambda: "mydiskofpoolone_2" in m.execute("virsh vol-list myPoolOne"))
        wait(lambda: "mydiskofpoolone_3" in m.execute("virsh vol-list myPoolOne"))

        args = self.createVm("subVmTest1")

        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydiskofpoolone_1 --target vdc --targetbus virtio".format(p1))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydiskofpoolone_2 --target vdd --targetbus virtio --persistent".format(p1))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydiskofpoolone_3 --target vde --targetbus virtio --persistent".format(p1))

        self.login_and_go("/machines")
        b.wait_in_text("body", "Virtual machines")
        self.waitVmRow("subVmTest1")

        # Wait for the login prompt before we try detaching disks - we need the OS to be fully responsive
        wait(lambda: "login as 'cirros' user" in self.machine.execute("cat {0}".format(args["logfile"])), delay=3)

        # Test detaching non permanent disk of a running domain
        b.wait_in_text("#vm-subVmTest1-state", "Running")
        self.goToVmPage("subVmTest1")

        b.wait_visible("#vm-subVmTest1-disks-vdc-device")
        b.click("#delete-subVmTest1-disk-vdc")
        b.wait_visible(".pf-c-modal-box")
        b.click(".pf-c-modal-box__footer button:contains(Remove)")
        b.wait_visible(".pf-c-modal-box__footer button.pf-m-in-progress")
        # When live-detaching disks the guest OS needs to cooperate so that we can
        # see the disk getting detached in the UI.
        # Wait until we see the login prompt before attempting operation that need
        # the OS for fully respond
        with b.wait_timeout(180):
            b.wait_not_present("#vm-subVmTest1-disks-vdc-device")
        b.wait_not_present(".pf-c-modal-box")

        # Test that detaching disk of a running domain will affect the
        # inactive configuration as well
        self.performAction("subVmTest1", "forceOff")
        b.wait_not_present("#vm-subVmTest1-disks-vdc-device")

        # Test detaching permanent disk of a stopped domain
        b.wait_visible("#vm-subVmTest1-disks-vdd-device")
        b.click("#delete-subVmTest1-disk-vdd")
        b.wait_visible(".pf-c-modal-box")
        b.click(".pf-c-modal-box__footer button:contains(Remove)")
        b.wait_not_present("#vm-subVmTest1-disks-vdd-device")
        b.wait_not_present(".pf-c-modal-box")

        # Test detaching several disks and the deletion dialog can be closed correctly
        m.execute("virsh vol-create-as myPoolOne diskVirtio --capacity 1M --format qcow2")
        m.execute("virsh vol-create-as myPoolOne diskSata --capacity 1M --format qcow2")
        wait(lambda: "diskVirtio" in m.execute("virsh vol-list myPoolOne"))
        wait(lambda: "diskSata" in m.execute("virsh vol-list myPoolOne"))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/diskVirtio --target vdf --targetbus virtio --persistent".format(p1))
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/diskSata --target sda --targetbus sata --persistent".format(p1))

        # Need to refresh when attaching disk for shutoff VM
        b.reload()
        b.enter_page('/machines')
        b.wait_in_text("body", "Virtual machines")
        b.wait_visible("#vm-subVmTest1-disks-sda-device")
        b.wait_visible("#vm-subVmTest1-disks-vdf-device")

        def removeDiskAndChecks(target):
            b.click("#delete-subVmTest1-disk-{}".format(target))
            b.wait_in_text("div[role=dialog]:contains(\"Remove Disk\")", target)
            b.click("div[role=dialog] button:contains(Remove)")
            # The deletion dialog should be closed
            b.wait_not_present("div[role=dialog]:contains(\"Remove Disk\")")
            # No error shown
            b.wait_not_present("div[aria-label=\"Danger Alert\"]")
            b.wait_not_present("#vm-subVmTest1-disks-{}-device".format(target))

        removeDiskAndChecks("sda")
        removeDiskAndChecks("vdf")

        # Test detaching disk of a paused domain
        m.execute("> {0}".format(args["logfile"]))  # clear logfile
        m.execute("virsh start subVmTest1")
        # Make sure that the VM booted normally before attempting to suspend it
        wait(lambda: "login as 'cirros' user" in m.execute("cat {0}".format(args["logfile"])), delay=3)
        m.execute("virsh suspend subVmTest1")
        b.wait_in_text("#vm-subVmTest1-state", "Paused")
        b.wait_attr("#delete-subVmTest1-disk-vde", "disabled", "")
        m.execute("virsh resume subVmTest1")
        wait(lambda: "login as 'cirros' user." in self.machine.execute("cat {0}".format(args["logfile"])), delay=3)

        # Test detaching of disk on non-persistent VM
        m.execute("virsh undefine subVmTest1")
        m.execute("virsh attach-disk --domain subVmTest1 --source {0}/mydiskofpoolone_1 --target vdc --targetbus virtio".format(p1))
        b.wait_visible("#vm-subVmTest1-disks-vdc-device")
        b.click("#delete-subVmTest1-disk-vdc")
        b.wait_visible(".pf-c-modal-box")
        b.click(".pf-c-modal-box__footer button:contains(Remove)")
        b.wait_not_present("#vm-subVmTest1-disks-vdc-device")
        b.wait_not_present(".pf-c-modal-box")


if __name__ == '__main__':
    test_main()
