diff --git a/genesis_devtools/cmd/cli.py b/genesis_devtools/cmd/cli.py index 17a0097..106076b 100644 --- a/genesis_devtools/cmd/cli.py +++ b/genesis_devtools/cmd/cli.py @@ -45,6 +45,7 @@ BOOTSTRAP_TAG = "bootstrap" LaunchModeType = tp.Literal["core", "element", "custom"] GC_CIDR = ipaddress.IPv4Network("10.20.0.0/22") +GC_BOOT_CIDR = ipaddress.IPv4Network("10.30.0.0/24") @click.group(invoke_without_command=True) @@ -498,14 +499,32 @@ def push_cmd( @click.option( "--cidr", default=GC_CIDR, - help="Network CIDR", + help="The main network CIDR", show_default=True, type=ipaddress.IPv4Network, ) @click.option( "--bridge", default=None, - help="Name of the linux bridge, it will be created if not set.", + help=( + "Name of the linux bridge for the main network, " + "it will be created if not set." + ), +) +@click.option( + "--boot-cidr", + default=GC_BOOT_CIDR, + help="The bootstrap network CIDR", + show_default=True, + type=ipaddress.IPv4Network, +) +@click.option( + "--boot-bridge", + default=None, + help=( + "Name of the linux bridge for the bootstrap network, " + "it will be created if not set." + ), ) @click.option( "-f", @@ -607,6 +626,8 @@ def bootstrap_cmd( stand_spec: str | None, cidr: ipaddress.IPv4Network, bridge: str | None, + boot_cidr: ipaddress.IPv4Network, + boot_bridge: str | None, force: bool, no_wait: bool, use_image_inplace: bool, @@ -676,6 +697,8 @@ def bootstrap_cmd( stand_spec=stand_spec, cidr=cidr, bridge=bridge, + boot_cidr=boot_cidr, + boot_bridge=boot_bridge, force=force, use_image_inplace=use_image_inplace, repository=repository, @@ -1305,6 +1328,8 @@ def _bootstrap_core( stand_spec: tp.Dict[str, tp.Any] | None, cidr: ipaddress.IPv4Network, bridge: str | None, + boot_cidr: ipaddress.IPv4Network, + boot_bridge: str | None, force: bool, use_image_inplace: bool, repository: str, @@ -1314,11 +1339,17 @@ def _bootstrap_core( logger.info("Starting genesis bootstrap in 'core' mode") net_name = utils.installation_net_name(name) - default_stand_network = stand_models.Network( + default_stand_main_network = stand_models.Network( name=bridge if bridge else net_name, cidr=cidr, managed_network=False if bridge else True, ) + boot_net_name = utils.installation_boot_net_name(name) + default_stand_boot_network = stand_models.Network( + name=boot_bridge if boot_bridge else boot_net_name, + cidr=boot_cidr, + managed_network=False if boot_bridge else True, + ) # Single bootstrap stand if stand_spec is None: @@ -1330,14 +1361,18 @@ def _bootstrap_core( use_image_inplace=use_image_inplace, cores=cores, memory=memory, - network=default_stand_network, + network=default_stand_main_network, + boot_network=default_stand_boot_network, bootstrap_name=bootstrap_domain_name, hypervisors=hypervisors, ) else: dev_stand = stand_models.Stand.from_spec(stand_spec) if dev_stand.network.is_dummy: - dev_stand.network = default_stand_network + dev_stand.network = default_stand_main_network + + if dev_stand.boot_network.is_dummy: + dev_stand.boot_network = default_stand_boot_network # Assign the image to bootstraps if it wasn't specified # in the specification. @@ -1368,8 +1403,13 @@ def _bootstrap_core( infra.delete_stand(stand) logger.info(f"Destroyed old genesis installation: {dev_stand.name}") - infra.create_stand(dev_stand, repository=repository) - logger.info("Launched genesis installation") + try: + infra.create_stand(dev_stand, repository=repository) + logger.info("Launched genesis installation") + except Exception: + infra.delete_stand(dev_stand) + logger.error(f"Failed to launch genesis installation {dev_stand.name}") + raise cidr = dev_stand.network.cidr logger.important( diff --git a/genesis_devtools/infra/driver/libvirt.py b/genesis_devtools/infra/driver/libvirt.py index 3116d4f..e7e704b 100644 --- a/genesis_devtools/infra/driver/libvirt.py +++ b/genesis_devtools/infra/driver/libvirt.py @@ -70,6 +70,24 @@ def _extract_net_from_bootstrap( name=name, cidr=cidr, managed_network=managed_network, dhcp=dhcp ) + def _extract_boot_net_from_bootstrap( + self, bootstrap: minidom.Document + ) -> models.Network: + try: + net = bootstrap.getElementsByTagName(vc.GENESIS_META_BOOT_NET_TAG)[ + 0 + ] + except Exception: + return models.Network.dummy() + + name = net.firstChild.nodeValue + cidr = ipaddress.IPv4Network(net.getAttribute("cidr")) + managed_network = bool(int(net.getAttribute("managed_network"))) + dhcp = bool(int(net.getAttribute("dhcp"))) + return models.Network( + name=name, cidr=cidr, managed_network=managed_network, dhcp=dhcp + ) + def _domain2bootstrap(self, domain: minidom.Document) -> models.Bootstrap: node = self._domain2node(domain) return models.Bootstrap.from_node(node) @@ -111,6 +129,9 @@ def list_stands(self) -> tp.List[models.Stand]: if node_type == "bootstrap": stand.bootstraps.append(self._domain2bootstrap(domain)) stand.network = self._extract_net_from_bootstrap(domain) + stand.boot_network = self._extract_boot_net_from_bootstrap( + domain + ) elif node_type == "baremetal": stand.baremetals.append(self._domain2node(domain)) else: @@ -136,9 +157,19 @@ def create_stand( ): raise ValueError(f"Some domain in stand {stand} already exists") - if stand.network.managed_network and libvirt.has_net(stand.network): - raise ValueError(f"Network {stand.network} already exists") + if stand.network.managed_network and libvirt.has_net( + stand.network.name + ): + raise ValueError(f"Network {stand.network.name} already exists") + + if stand.boot_network.managed_network and libvirt.has_net( + stand.boot_network.name + ): + raise ValueError( + f"Network {stand.boot_network.name} already exists" + ) + # Main network for ordinary communication if stand.network.managed_network: libvirt.create_nat_network( name=stand.network.name, @@ -146,9 +177,15 @@ def create_stand( dhcp_enabled=stand.network.dhcp, ) + # Isolated network for bootstrap procedure + if stand.boot_network.managed_network: + libvirt.create_isolated_network( + name=stand.boot_network.name, + ) + # Prepare config drive for the bootstrap node spec = { - "schema_version": 1, + "schema_version": 2, "stand": dataclasses.asdict(stand), **extra_data, } @@ -181,6 +218,17 @@ def create_stand( "dhcp": int(stand.network.dhcp), }, ), + self._tag( + vc.GENESIS_META_BOOT_NET_TAG, + stand.boot_network.name, + { + "cidr": str(stand.boot_network.cidr), + "managed_network": int( + stand.boot_network.managed_network + ), + "dhcp": int(stand.boot_network.dhcp), + }, + ), ) libvirt.create_domain( @@ -190,10 +238,7 @@ def create_stand( cores=bootstrap.cores, memory=bootstrap.memory, disks=bootstrap.disks, - network=stand.network.name, - net_type=( - "network" if stand.network.managed_network else "bridge" - ), + networks=(stand.network, stand.boot_network), meta_tags=tags, config_drive=config_drive_path, ) @@ -214,10 +259,8 @@ def create_stand( use_image_inplace=node.use_image_inplace, cores=node.cores, memory=node.memory, - network=stand.network.name, - net_type=( - "network" if stand.network.managed_network else "bridge" - ), + # Put new node to the boot network + networks=(stand.boot_network,), meta_tags=tags, disks=node.disks, boot="network", @@ -226,7 +269,9 @@ def create_stand( def delete_stand(self, stand: models.Stand) -> None: """Delete the stand.""" for node in itertools.chain(stand.bootstraps, stand.baremetals): - libvirt.destroy_domain(node.name) + if libvirt.has_domain(node.name): + libvirt.destroy_domain(node.name) - if stand.network.managed_network: - libvirt.destroy_net(stand.network.name) + for net in (stand.network.name, stand.boot_network.name): + if libvirt.has_net(net): + libvirt.destroy_net(net) diff --git a/genesis_devtools/infra/libvirt/constants.py b/genesis_devtools/infra/libvirt/constants.py index 1173e2b..5d7eadc 100644 --- a/genesis_devtools/infra/libvirt/constants.py +++ b/genesis_devtools/infra/libvirt/constants.py @@ -23,5 +23,6 @@ GENESIS_META_MEM_TAG = "genesis:mem" GENESIS_META_IMAGE_TAG = "genesis:image" GENESIS_META_NET_TAG = "genesis:network" +GENESIS_META_BOOT_NET_TAG = "genesis:boot_network" BootMode = tp.Literal["hd", "network"] diff --git a/genesis_devtools/infra/libvirt/libvirt.py b/genesis_devtools/infra/libvirt/libvirt.py index 6d69b7f..d147373 100644 --- a/genesis_devtools/infra/libvirt/libvirt.py +++ b/genesis_devtools/infra/libvirt/libvirt.py @@ -23,6 +23,7 @@ import typing as tp import uuid as sys_uuid +from genesis_devtools.stand import models from genesis_devtools import constants as c from genesis_devtools.infra.libvirt import constants as vc @@ -74,7 +75,7 @@ - {net_iface} + {net_ifaces} @@ -124,6 +125,14 @@ """ +isolated_network_no_dhcp_template = """ + + {name} + + +""" + + network_iface_template = """ @@ -163,7 +172,7 @@ def list_domains( ) -> tp.List[str]: """List all domains.""" out = subprocess.check_output( - f"sudo virsh list --{state} --name", shell=True + ["sudo", "virsh", "list", f"--{state}", "--name"] ) out = out.decode().strip() names = [o for o in out.split("\n") if o] @@ -175,7 +184,7 @@ def list_domains( # Find all domains with the corresponding meta tag domains = [] for name in names: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() if meta_tag in out: domains.append(name) @@ -188,7 +197,7 @@ def list_xml_domains( ) -> tp.List[str]: """List all domains.""" out = subprocess.check_output( - f"sudo virsh list --{state} --name", shell=True + ["sudo", "virsh", "list", f"--{state}", "--name"] ) out = out.decode().strip() names = [o for o in out.split("\n") if o] @@ -197,7 +206,7 @@ def list_xml_domains( # Find all domains with the corresponding meta tag domains = [] for name in names: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() if meta_tag and meta_tag in out: domains.append(out) @@ -214,7 +223,7 @@ def is_active_domain(name: str) -> bool: def list_nets(): """List all networks.""" out = subprocess.check_output( - "sudo virsh net-list --all --name", shell=True + ["sudo", "virsh", "net-list", "--all", "--name"] ) out = out.decode().strip() return out.split("\n") @@ -223,12 +232,32 @@ def list_nets(): def list_pool(): """List all pools.""" out = subprocess.check_output( - "sudo virsh pool-list --all --name", shell=True + ["sudo", "virsh", "pool-list", "--all", "--name"] ) out = out.decode().strip() return out.split("\n") +def define_network(name: str, net_xml: str): + with tempfile.TemporaryDirectory() as temp_dir: + network_path = os.path.join(temp_dir, f"{name}.xml") + with open(network_path, "w") as f: + f.write(net_xml) + + subprocess.check_call( + ["sudo", "virsh", "net-define", network_path], + stdout=subprocess.DEVNULL, + ) + subprocess.check_call( + ["sudo", "virsh", "net-start", name], + stdout=subprocess.DEVNULL, + ) + subprocess.check_call( + ["sudo", "virsh", "net-autostart", name], + stdout=subprocess.DEVNULL, + ) + + def create_nat_network( name: str, cidr: ipaddress.IPv4Network, dhcp_enabled: bool = True ): @@ -245,32 +274,20 @@ def create_nat_network( else: network = nat_network_no_dhcp_template.format(**net_params) - with tempfile.TemporaryDirectory() as temp_dir: - network_path = os.path.join(temp_dir, f"{name}.xml") - with open(network_path, "w") as f: - f.write(network) + define_network(name, network) - subprocess.run( - f"sudo virsh net-define {network_path} 1>/dev/null", - shell=True, - check=True, - ) - subprocess.run( - f"sudo virsh net-start {name} 1>/dev/null", shell=True, check=True - ) - subprocess.run( - f"sudo virsh net-autostart {name} 1>/dev/null", - shell=True, - check=True, - ) + +def create_isolated_network(name: str): + net_params = {"name": name} + network = isolated_network_no_dhcp_template.format(**net_params) + define_network(name, network) def create_domain( name: str, cores: str, memory: int, - network: str, - net_type: str = "network", + networks: tp.Collection[models.Network], pool: str = c.LIBVIRT_DEF_POOL_PATH, image: str | None = None, use_image_inplace: bool = False, @@ -281,6 +298,7 @@ def create_domain( ): uuid = str(sys_uuid.uuid4()) disks_xml = "" + ifaces_xml = "" disk_paths = [] index = 0 @@ -314,7 +332,7 @@ def create_domain( disk_name = f"{uuid}-{i}.qcow2" disk_path = os.path.join(pool, disk_name) disk_paths.append(disk_path) - subprocess.run( + subprocess.check_call( [ "sudo", "qemu-img", @@ -324,7 +342,6 @@ def create_domain( disk_path, f"{disk}G", ], - check=True, stdout=subprocess.DEVNULL, ) disks_xml += disk_template.format( @@ -333,10 +350,12 @@ def create_domain( image_format="qcow2", ) - if net_type == "network": - network_iface = network_iface_template.format(network=network) - else: - network_iface = bridge_iface_template.format(network=network) + for network in networks: + if network.managed_network: + network_iface = network_iface_template.format(network=network.name) + else: + network_iface = bridge_iface_template.format(network=network.name) + ifaces_xml += network_iface meta_tags_xml = "\n\t\t".join(tag for tag in meta_tags) @@ -351,6 +370,7 @@ def create_domain( subprocess.run( ["sudo", "cp", config_drive, config_drive_path], check=True, + stdout=subprocess.DEVNULL, ) disks_xml += cdrom_template.format( config_drive_path=config_drive_path, @@ -361,7 +381,7 @@ def create_domain( name=name, cores=cores, memory=memory, - net_iface=network_iface, + net_ifaces=ifaces_xml, disks=disks_xml, uuid=uuid, meta_tags=meta_tags_xml, @@ -374,24 +394,22 @@ def create_domain( f.write(domain) try: - subprocess.run( - f"sudo virsh define {domain_path} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "define", domain_path], + stdout=subprocess.DEVNULL, ) - subprocess.run( - f"sudo virsh start {name} 1>/dev/null", shell=True, check=True + subprocess.check_call( + ["sudo", "virsh", "start", name], + stdout=subprocess.DEVNULL, ) except Exception: # Unable to create domain, delete disks for disk_path in disk_paths: - subprocess.run( - f"sudo rm -f {disk_path}", shell=True, check=True - ) + subprocess.check_call(["sudo", "rm", "-f", disk_path]) def get_domain_ip(name: str) -> tp.Optional[str]: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() mac_addresses = re.findall(r" tp.Optional[str]: # Actually it's not right solution but for simplicity keep it. for mac, net in zip(mac_addresses, networs): out = subprocess.check_output( - f"sudo virsh net-dhcp-leases {net}", - shell=True, + ["sudo", "virsh", "net-dhcp-leases", net], ) out = out.decode().strip() for line in out.split("\n"): @@ -413,7 +430,7 @@ def get_domain_ip(name: str) -> tp.Optional[str]: def get_domain_disk(name: str) -> str | None: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() # The simplest implementation, take first disk @@ -422,7 +439,7 @@ def get_domain_disk(name: str) -> str | None: def get_domain_disks(name: str) -> tp.List[str]: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() # The simplest implementation, take first disk @@ -443,10 +460,9 @@ def destroy_domain(name: str) -> None: if is_active_domain(name): try: - subprocess.run( - f"sudo virsh destroy {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "destroy", name], + stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError: # Nothing to do, the domain is already destroyed @@ -464,28 +480,27 @@ def destroy_domain(name: str) -> None: # Remove the disk for disk_path in domain_disks: - subprocess.run( - f"sudo rm -f {disk_path} 1>/dev/null", shell=True, check=True + subprocess.check_call( + ["sudo", "rm", "-f", disk_path], + stdout=subprocess.DEVNULL, ) def destroy_net(name: str) -> None: """Delete network.""" try: - subprocess.run( - f"sudo virsh net-destroy {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "net-destroy", name], + stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError: # Nothing to do, the network is already destroyed pass try: - subprocess.run( - f"sudo virsh net-undefine {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "net-undefine", name], + stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError: # Nothing to do, the network is already undefined @@ -493,7 +508,7 @@ def destroy_net(name: str) -> None: def domain_xml(name: str) -> str: - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) return out.decode().strip() @@ -501,7 +516,7 @@ def backup_domain(name: str, backup_path: str) -> None: disks = get_domain_disks(name) # Save domain xml - out = subprocess.check_output(f"sudo virsh dumpxml {name}", shell=True) + out = subprocess.check_output(["sudo", "virsh", "dumpxml", name]) out = out.decode().strip() with open(os.path.join(backup_path, "domain.xml"), "w") as f: @@ -512,64 +527,83 @@ def backup_domain(name: str, backup_path: str) -> None: for disk in disks: disk_name = os.path.basename(disk) backup_disk_path = os.path.join(backup_path, disk_name) - subprocess.run( - f"sudo cp {disk} {backup_disk_path}", shell=True, check=True - ) + subprocess.check_call(["sudo", "cp", disk, backup_disk_path]) return # Active domain try: - subprocess.run( - f"sudo virsh suspend {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "suspend", name], + stdout=subprocess.DEVNULL, ) for disk in disks: disk_name = os.path.basename(disk) backup_disk_path = os.path.join(backup_path, disk_name) - subprocess.run( - f"sudo cp {disk} {backup_disk_path}", shell=True, check=True - ) + subprocess.check_call(["sudo", "cp", disk, backup_disk_path]) finally: - subprocess.run( - f"sudo virsh resume {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "resume", name], + stdout=subprocess.DEVNULL, ) def create_snapshot(domain: str, snap_name: str = "snapshot") -> None: # Create a snapshot - subprocess.check_output( - f"sudo virsh snapshot-create-as {domain} {snap_name} " - "--disk-only --quiesce --atomic 1>/dev/null", - shell=True, + subprocess.check_call( + [ + "sudo", + "virsh", + "snapshot-create-as", + domain, + snap_name, + "--disk-only", + "--quiesce", + "--atomic", + ], + stdout=subprocess.DEVNULL, ) def delete_snapshot(domain: str, snap_name: str = "snapshot") -> None: - subprocess.check_output( - f"sudo virsh snapshot-delete {domain} {snap_name} " - "--metadata 1>/dev/null", - shell=True, + subprocess.check_call( + [ + "sudo", + "virsh", + "snapshot-delete", + domain, + snap_name, + "--metadata", + ], + stdout=subprocess.DEVNULL, ) def merge_disk_snapshot( domain: str, device: str, disk_path: str, snapshot_path: str ) -> None: - subprocess.check_output( - f"sudo virsh blockcommit --domain {domain} {device} --top " - f"{snapshot_path} --base {disk_path} --wait --pivot 1>/dev/null", - shell=True, + subprocess.check_call( + [ + "sudo", + "virsh", + "blockcommit", + "--domain", + domain, + device, + "--top", + snapshot_path, + "--base", + disk_path, + "--wait", + "--pivot", + ], + stdout=subprocess.DEVNULL, ) def resume_domain(name: str) -> None: - subprocess.run( - f"sudo virsh resume {name} 1>/dev/null", - shell=True, - check=True, + subprocess.check_call( + ["sudo", "virsh", "resume", name], + stdout=subprocess.DEVNULL, ) diff --git a/genesis_devtools/stand/models.py b/genesis_devtools/stand/models.py index 6d391c4..edd81a7 100644 --- a/genesis_devtools/stand/models.py +++ b/genesis_devtools/stand/models.py @@ -105,7 +105,12 @@ def is_valid(self, network: Network) -> bool: @dataclasses.dataclass class Stand: + # Main network of the installation. After bootstrap + # procedure a node will be connected to this network. network: Network + # Network used for the bootstrapping procedure. + # It's an isolated private network. + boot_network: Network bootstraps: list[Bootstrap] baremetals: list[Node] hypervisors: list[Hypervisor] = dataclasses.field(default_factory=list) @@ -137,6 +142,7 @@ def single_bootstrap_stand( image: str, use_image_inplace: bool, network: Network, + boot_network: Network, cores: int = 1, memory: int = 1024, name: str = "dev-stand", @@ -146,6 +152,7 @@ def single_bootstrap_stand( return cls( name=name, network=network, + boot_network=boot_network, bootstraps=[ Bootstrap( name=bootstrap_name, @@ -161,16 +168,23 @@ def single_bootstrap_stand( @classmethod def empty_stand( - cls, name: str = "dev-stand", network: Network | None = None + cls, + name: str = "dev-stand", + network: Network | None = None, + boot_network: Network | None = None, ) -> Stand: if network is None: network = Network.dummy() + if boot_network is None: + boot_network = Network.dummy() + return cls( name=name, bootstraps=[], baremetals=[], network=network, + boot_network=boot_network, hypervisors=[], ) @@ -187,10 +201,16 @@ def from_spec(cls, spec: dict[str, tp.Any]) -> Stand: else: network = Network.from_spec(spec.pop("network")) + if "boot_network" not in spec: + boot_network = Network.dummy() + else: + boot_network = Network.from_spec(spec.pop("boot_network")) + return cls( bootstraps=bootstraps, baremetals=baremetals, network=network, + boot_network=boot_network, hypervisors=[ Hypervisor.from_spec(h) for h in spec.pop("hypervisors", []) ], diff --git a/genesis_devtools/utils.py b/genesis_devtools/utils.py index 20ce850..666fac9 100644 --- a/genesis_devtools/utils.py +++ b/genesis_devtools/utils.py @@ -97,6 +97,10 @@ def installation_net_name(name: str) -> str: return f"{name}-net" +def installation_boot_net_name(name: str) -> str: + return f"{name}-boot-net" + + def installation_bootstrap_name(name: str) -> str: return f"{name}-bootstrap"