diff --git a/ctf/models.py b/ctf/models.py index 7319e99..3caa404 100644 --- a/ctf/models.py +++ b/ctf/models.py @@ -102,7 +102,7 @@ class TrackInstance(BaseModel): ipv6: str | None = None config: InstanceConfig devices: list[InstanceDevice] - wait_for: InstanceWaitFor | None = None + wait_for: list[InstanceWaitFor] | None = None services: list[InstanceService] diff --git a/ctf/new.py b/ctf/new.py index 3f42ab5..52d9155 100644 --- a/ctf/new.py +++ b/ctf/new.py @@ -15,10 +15,11 @@ class Template(StrEnum): + INFRA_SKELETON = "infra-skeleton" + TRACK_YAML_ONLY = "track-yaml-only" + FILES_ONLY = "files-only" APACHE_PHP = "apache-php" PYTHON_SERVICE = "python-service" - FILES_ONLY = "files-only" - TRACK_YAML_ONLY = "track-yaml-only" RUST_WEBSERVICE = "rust-webservice" WINDOWS_VM = "windows-vm" @@ -40,7 +41,7 @@ def new( help="Template to use for the track.", prompt="Template to use for the track.", ), - ] = Template.APACHE_PHP, + ] = Template.INFRA_SKELETON, force: Annotated[ bool, typer.Option( @@ -55,6 +56,14 @@ def new( help="If a build container is required.", ), ] = False, + with_virtual_machine: Annotated[ + bool, + typer.Option( + "--vm", + "--with-virtual-machine", + help="If a virtual machine is required.", + ), + ] = False, ) -> None: LOG.info(msg=f"Creating a new track: {name}") if not re.match(pattern=r"^[a-z][a-z0-9\-]{0,61}[a-z0-9]$", string=name): @@ -130,6 +139,7 @@ def new( "is_windows": template == Template.WINDOWS_VM, "template": template.value, "with_build": with_build_container, + "with_virtual_machine": with_virtual_machine, } ) with open( @@ -206,6 +216,7 @@ def new( "ipv6_subnet": ipv6_subnet, "full_ipv6_address": full_ipv6_address, "with_build": with_build_container, + "with_virtual_machine": with_virtual_machine, "is_windows": template == Template.WINDOWS_VM, } ) @@ -245,7 +256,11 @@ def new( track_template = env.get_template(name=os.path.join(template, "deploy.yaml.j2")) render = track_template.render( - data={"name": name, "with_build": with_build_container} + data={ + "name": name, + "with_build": with_build_container, + "with_virtual_machine": with_virtual_machine, + } ) with open( file=(p := os.path.join(ansible_directory, "deploy.yaml")), @@ -283,6 +298,7 @@ def new( data={ "name": name, "with_build": with_build_container, + "with_virtual_machine": with_virtual_machine, "is_windows": template == Template.WINDOWS_VM, } ) diff --git a/ctf/templates/init/.deploy/cleanup.yaml b/ctf/templates/init/.deploy/cleanup.yaml index d3517a0..c5cf244 100644 --- a/ctf/templates/init/.deploy/cleanup.yaml +++ b/ctf/templates/init/.deploy/cleanup.yaml @@ -1,5 +1,5 @@ - name: Pre-deployment system cleanup - hosts: all,!build,!windows + hosts: all,!build,!windows,!linux-incus-vm order: shuffle gather_facts: false any_errors_fatal: true diff --git a/ctf/templates/init/schemas/track.yaml.json b/ctf/templates/init/schemas/track.yaml.json index 871e559..996d9af 100644 --- a/ctf/templates/init/schemas/track.yaml.json +++ b/ctf/templates/init/schemas/track.yaml.json @@ -142,7 +142,7 @@ } }, "wait_for": { - "type": "object", + "type": "array", "description": "Optional wait condition for instance startup.", "additionalProperties": true }, diff --git a/ctf/templates/new/apache-php/deploy.yaml.j2 b/ctf/templates/new/apache-php/deploy.yaml.j2 index 363172a..1ac937d 100644 --- a/ctf/templates/new/apache-php/deploy.yaml.j2 +++ b/ctf/templates/new/apache-php/deploy.yaml.j2 @@ -37,9 +37,9 @@ path: "/var/www/html/index.html" state: absent -# Run this only on the `{{ data.name }}` host. Sets up apache2 and php. -- name: Specific tasks for "{{ data.name }}" - hosts: "{{ data.name }}" +# Run this only on the `{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}` host. Sets up apache2 and php. +- name: Specific tasks for "{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}" + hosts: "{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}" tasks: - name: Copy the main site file (index.php) ansible.builtin.template: diff --git a/ctf/templates/new/common/inventory.j2 b/ctf/templates/new/common/inventory.j2 index 346c687..07ff8cf 100644 --- a/ctf/templates/new/common/inventory.j2 +++ b/ctf/templates/new/common/inventory.j2 @@ -1,6 +1,7 @@ # This YAML file defines all machines that Ansible needs to know about to run playbooks and configure machines. +# CHANGE SINCE 5.0.1: Virtual machines are deployed in another remote/project in production. Added/Modified a few variables with templates to enable that. all: - hosts:{% if not data.is_windows %} + hosts:{% if not data.is_windows and not data.with_virtual_machine %} # The following line defines how this machine will be referred to in Ansible scripts. {{ data.name }}: # This one tells Ansible that this host is reached using incus, and the name of the machine in incus is `{{ data.name }}`. @@ -15,6 +16,7 @@ all: # Do not change these. ansible_connection: community.general.incus ansible_user: root + # If you have virtual machines, do not alter this line. ansible_incus_remote: "{{ '{{' }} ansible_incus_container_remote if ansible_incus_container_remote else 'local' {{ '}}' }}" # Name of your track. Do not change. ansible_incus_project: {{ data.name }} @@ -22,23 +24,45 @@ all: # Add variables if needed here. {% if data.with_build %} # This section is needed if you need a build container. It's a group of hosts regrouped under the name "build" which MUST remain the same. -# The group "build" is removed from the "cleanup.yaml" and "common.yaml", which is why you should not change it. +# The group "build" is removed from the "cleanup.yaml" and "common.yaml", which is why it should remain the same. build: hosts: # The following line defines how this machine will be referred to in "build.yaml" Ansible script. build-container: # The name must be the same as the previous line. ansible_incus_host: build-container +{% endif %}{% if data.with_virtual_machine %} +# This section is needed if you need Linux virtual machines. It's a group of hosts regrouped under the name "linux-incus-vm" which MUST remain the same. +# The group "linux-incus-vm" is partially included in the "cleanup.yaml" playbook, which is why it should remain the same. +# The name of any VM must begin with `team-orig-` for production +linux-incus-vm: + hosts: + # The following line defines how this machine will be referred to in Ansible scripts. KEEP THE `team-orig-` prefix. + team-orig-{{ data.name }}: + # This one tells Ansible that this host is reached using incus, and the name of the machine in incus is `team-orig-{{ data.name }}`. + ansible_incus_host: team-orig-{{ data.name }} + vars: + # This variable is used to tell Ansible that the hosts are Windows hosts and require a PowerShell shell. + ansible_connection: community.general.incus + ansible_user: root + # For virtual machines, the remote and project changes in production. + ansible_incus_remote: "{{ '{{' }} ansible_incus_container_remote if ansible_incus_vm_remote else ansible_incus_container_remote {{ '}}' }}" + ansible_incus_project: "{{ '{{' }} ansible_incus_vm_project if ansible_incus_vm_project else ansible_incus_project {{ '}}' }}" {% endif %}{% if data.is_windows %} # This section is needed if you need Windows virtual machines. It's a group of hosts regrouped under the name "windows" which MUST remain the same. -# The group "windows" is removed from the "cleanup.yaml" and "common.yaml", which is why you should not change it. +# The group "windows" is removed from the "cleanup.yaml" and "common.yaml", which is why it should remain the same. +# The name of any VM must begin with `team-orig-` for production windows: hosts: - # The following line defines how this machine will be referred to in Ansible scripts. - {{ data.name }}: - # This one tells Ansible that this host is reached using incus, and the name of the machine in incus is `{{ data.name }}`. - ansible_incus_host: {{ data.name }} + # The following line defines how this machine will be referred to in Ansible scripts. KEEP THE `team-orig-` prefix. + team-orig-{{ data.name }}: + # This one tells Ansible that this host is reached using incus, and the name of the machine in incus is `team-orig-{{ data.name }}`. + ansible_incus_host: team-orig-{{ data.name }} vars: # This variable is used to tell Ansible that the hosts are Windows hosts and require a PowerShell shell. ansible_shell_type: powershell + ansible_user: administrator + # For virtual machines, the remote and project changes in production. + ansible_incus_remote: "{{ '{{' }} ansible_incus_container_remote if ansible_incus_vm_remote else ansible_incus_container_remote {{ '}}' }}" + ansible_incus_project: "{{ '{{' }} ansible_incus_vm_project if ansible_incus_vm_project else ansible_incus_project {{ '}}' }}" {% endif %} diff --git a/ctf/templates/new/common/main.tf.j2 b/ctf/templates/new/common/main.tf.j2 index 7ace05b..9b769c8 100644 --- a/ctf/templates/new/common/main.tf.j2 +++ b/ctf/templates/new/common/main.tf.j2 @@ -13,6 +13,37 @@ resource "incus_project" "this" { } } +# If you need network ACL to be applied after Ansible, use this section. +# The example below allows TCP port 8000 ingress and allows any for shell.ctf egress. +/* Uncomment if using ACL (there is another line to uncomment in the network) +resource "incus_network_acl" "this" { + count = var.already_deployed ? 1 : 0 + + remote = var.incus_remote + project = incus_project.this.name + + name = local.track.name + description = "Allow egress to shell.ctf and ingress on 8000." + + egress = [ + { + action = "allow" + destination = "9000:6666:6666:6666::/64" + state = "enabled" + } + ] + + ingress = [ + { + action = "allow" + protocol = "tcp" + destination_port = "8000" + state = "enabled" + } + ] +} +*/ + # AUTOGENERATED - No need to change this section # resource "incus_network" "this" { remote = var.incus_remote @@ -21,27 +52,48 @@ resource "incus_network" "this" { name = substr(local.track.name, 0, 15) description = "Network for challenges in the ${local.track.name} track" - config = { - "ipv4.address" = var.deploy == "production" ? "none" : null + config = merge({ "ipv6.address" = "{{ data.ipv6_subnet }}::1/64" - "ipv6.nat" = var.deploy == "production" ? "false" : "true" - } + }, + /* Uncomment if using ACL + var.already_deployed ? { + "security.acls" = incus_network_acl.this[0].name + } : {}, + */ + var.deploy == "production" ? { + "ipv4.address" = "none" + "ipv6.nat" = "false" + # Uncomment only when using virtual machines. + {% if not data.is_windows and not data.with_virtual_machine %}# {% endif %}"bridge.external_interfaces" = "enp1s0" # This must be changed by the infra team near the CTF. + } + : + { + "ipv6.nat" = "true" + } + ) } -# Some settings can be changed here # +# Some settings can be changed here. +# Use only when you have containers (majority of the time). resource "incus_profile" "this" { remote = var.incus_remote project = incus_project.this.name - name = {% if data.is_windows %}"windows-vm"{% else %}"containers"{% endif %} - description = "Default profile for {% if data.is_windows %}Windows virtual machine{% else %}containers{% endif %} in the ${local.track.name} track" + name = "containers" + description = "Default profile for containers in the ${local.track.name} track." - config = { + config = merge({ # These limits should only be adjusted if you NEED more resources. "limits.cpu" = "2" - "limits.memory" = {% if data.is_windows %}"3GiB"{% else %}"256MiB" - "limits.processes" = "2000"{% endif %} - } + "limits.memory" = "256MiB" + "limits.processes" = "2000" + }, + var.deploy == "production" ? + { + "environment.http_proxy" = "http://proxy.ctf-int.internal.nsec.io:3128" + "environment.https_proxy" = "http://proxy.ctf-int.internal.nsec.io:3128" + "environment.no_proxy" = "127.0.0.1,::1,localhost" + } : {}) device { name = "root" @@ -51,21 +103,147 @@ resource "incus_profile" "this" { "pool" = "default" "path" = "/" # This limit should only be adjusted if you NEED more resources. - "size" = {% if data.is_windows %}"32GiB"{% else %}"1GiB"{% endif %} + "size" = "1GiB" + } + } +} + +# These values are ONLY used for virtual machines. No need to tweak them if you only have containers. +locals { + vm_profiles = { + "config" = { + # These limits should only be adjusted if you NEED more resources. + "limits.cpu" = "2" + "boot.autorestart" = "true" + "security.secureboot" = "false" } } + linux_vm_profiles = { + "config" = merge({ + # These limits should only be adjusted if you NEED more resources. + "limits.memory" = "512MiB" + }, + var.deploy == "production" ? + { + "environment.http_proxy" = "http://proxy.ctf-int.internal.nsec.io:3128" + "environment.https_proxy" = "http://proxy.ctf-int.internal.nsec.io:3128" + "environment.no_proxy" = "127.0.0.1,::1,localhost" + } : {}), + "disk_size" = "16GiB" + } + windows_vm_profiles = { + "config" = { + # These limits should only be adjusted if you NEED more resources. + "limits.memory" = "1GiB" + }, + "disk_size" = "32GiB" + } } # AUTOGENERATED - No need to change this section # -resource "incus_instance" "this" { +resource "incus_profile" "linux_vm_local" { remote = var.incus_remote project = incus_project.this.name + name = "linux-vm" + description = "Default profile for Linux virtual machine in the ${local.track.name} track." + + config = merge(local.vm_profiles["config"], local.linux_vm_profiles["config"]) + + device { + name = "root" + type = "disk" + + properties = { + "pool" = "default" + "path" = "/" + "size" = local.linux_vm_profiles["disk_size"] + } + } +} + +# AUTOGENERATED - No need to change this section # +# This resource block should reflect one to one the previous block with a few exceptions. +resource "incus_profile" "linux_vm_remote" { + count = (var.incus_vm_remote != null && var.incus_vm_project != null) ? 1 : 0 + + remote = var.incus_vm_remote + project = var.incus_vm_project + + name = substr("${local.track.name}-linux-vm", 0, 64) # Except this field + description = "Default profile for Linux virtual machine in the ${local.track.name} track." + + config = merge(local.vm_profiles["config"], local.linux_vm_profiles["config"]) + + device { + name = "root" + type = "disk" + + properties = { + "pool" = "remote" # Except this field + "path" = "/" + "size" = local.linux_vm_profiles["disk_size"] + } + } +} + +# AUTOGENERATED - No need to change this section # +resource "incus_profile" "windows_vm_local" { + remote = var.incus_remote + project = incus_project.this.name + + name = "windows-vm" + description = "Default profile for Windows virtual machine in the ${local.track.name} track." + + config = merge(local.vm_profiles["config"], local.windows_vm_profiles["config"]) + + device { + name = "root" + type = "disk" + + properties = { + "pool" = "default" + "path" = "/" + "size" = local.windows_vm_profiles["disk_size"] + } + } +} + +# AUTOGENERATED - No need to change this section # +# This resource block should reflect one to one the previous block with a few exceptions. +resource "incus_profile" "windows_vm_remote" { + count = (var.incus_vm_remote != null && var.incus_vm_project != null) ? 1 : 0 + + remote = var.incus_vm_remote + project = var.incus_vm_project + + name = substr("${local.track.name}-windows-vm", 0, 64) # Except this field + description = "Default profile for Windows virtual machine in the ${local.track.name} track." + + config = merge(local.vm_profiles["config"], local.windows_vm_profiles["config"]) + + device { + name = "root" + type = "disk" + + properties = { + "pool" = "remote" # Except this field + "path" = "/" + "size" = local.windows_vm_profiles["disk_size"] + } + } +} + +# AUTOGENERATED - No need to change this section # +resource "incus_instance" "this" { for_each = { for k, v in try(local.track.instances, {}) : k => v if !can(v.is_build_container) || v.is_build_container == var.build_container } + remote = var.incus_vm_remote != null && each.value["type"] == "virtual-machine" ? var.incus_vm_remote : var.incus_remote + project = var.incus_vm_project != null && each.value["type"] == "virtual-machine" ? var.incus_vm_project : incus_project.this.name + name = each.key description = each.value["description"] @@ -74,23 +252,22 @@ resource "incus_instance" "this" { image = each.value["image"] profiles = [ for profile in coalesce(try(each.value["profiles"], []), []) : - profile == incus_profile.this.name ? incus_profile.this.name : profile + profile == incus_profile.this.name ? + incus_profile.this.name : + ( + endswith(profile, "linux-vm") ? (var.incus_vm_remote != null ? incus_profile.linux_vm_remote[0].name : incus_profile.linux_vm_local.name) : + endswith(profile, "windows-vm") ? (var.incus_vm_remote != null ? incus_profile.windows_vm_remote[0].name : incus_profile.windows_vm_local.name) : + profile + ) ] - config = merge( - coalesce(try(each.value["config"], {}), {}), - { - "environment.http_proxy" = var.deploy == "production" ? "http://proxy.ctf-int.internal.nsec.io:3128" : null - "environment.https_proxy" = var.deploy == "production" ? "http://proxy.ctf-int.internal.nsec.io:3128" : null - "environment.no_proxy" = var.deploy == "production" ? "127.0.0.1,::1,localhost" : null - } - ) + config = coalesce(try(each.value["config"], {}), {}) dynamic "device" { for_each = coalesce(try(each.value["devices"], []), []) content { - name = device.value.name - type = device.value.type + name = device.value["name"] + type = device.value["type"] properties = device.value["properties"] } @@ -101,12 +278,17 @@ resource "incus_instance" "this" { type = "nic" properties = merge({ - "network" = incus_network.this.name - "name" = "eth0" + "name" = "eth0" }, try(trimspace(each.value["hwaddr"]), "") != "" ? { hwaddr = trimspace(each.value["hwaddr"]) - } : {} + } : {}, + var.incus_vm_remote != null && each.value["type"] == "virtual-machine" ? { + "network" = "TEAM-ORIG-${local.track.name}" + "security.promiscuous" = "true" + } : { + "network" = incus_network.this.name + } ) } @@ -117,7 +299,7 @@ resource "incus_instance" "this" { dynamic "wait_for" { for_each = coalesce(try(each.value["wait_for"], []), []) content { - type = wait_for.value.type + type = wait_for.value["type"] } } } @@ -145,17 +327,19 @@ resource "incus_network_zone_record" "this" { } # If you need to manually add DNS records, here is an example. -#resource "incus_network_zone_record" "sub" { -# remote = var.incus_remote -# -# zone = "ctf" -# -# name = "sub.chal5.hackademy" -# description = local.track.instances["challenge5"]["description"] -# -# entry { -# type = "AAAA" -# ttl = 3600 -# value = "${local.track.instances["challenge5"]["ipv6"]}" -# } -#} +/* Uncomment if you need subdomains +resource "incus_network_zone_record" "sub" { + remote = var.incus_remote + + zone = "ctf" + + name = "sub.chal5.hackademy" + description = local.track.instances["challenge5"]["description"] + + entry { + type = "AAAA" + ttl = 3600 + value = "${local.track.instances["challenge5"]["ipv6"]}" + } +} +*/ diff --git a/ctf/templates/new/common/track.yaml.j2 b/ctf/templates/new/common/track.yaml.j2 index 9eb16af..331a1dc 100644 --- a/ctf/templates/new/common/track.yaml.j2 +++ b/ctf/templates/new/common/track.yaml.j2 @@ -20,29 +20,27 @@ contacts: instances: {} {% else %} instances: - {{ data.name }}: + {% if data.is_windows or data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}: image: {% if data.is_windows %}"CHANGE_ME" # Change to the Windows image location. Refer to the ctf-script README to know how to create an image in Incus.{% else %}"images:ubuntu/24.04"{% endif %} - profiles:{% if not data.is_windows %} - - "default"{% endif %} - - {% if data.is_windows %}"windows-vm"{% else %}"containers"{% endif %} - type: {% if data.is_windows %}"virtual-machine"{% else %}"container"{% endif %} - description: "CHANGE_MEMain apache + PHP website" + profiles: + - "{% if data.is_windows %}windows-vm{% elif data.with_virtual_machine %}linux-vm{% else %}containers{% endif %}" + type: {% if data.is_windows or data.with_virtual_machine %}"virtual-machine"{% else %}"container"{% endif %} + description: "CHANGE_ME" # This second half of a MAC address was automatically generated. The first half is just below hwaddr: "{{ data.hardware_address }}" # This is the subdomain value for this machine. Do NOT include .ctf, it will be automatically added. # For example, if the value of the record is a.b.c, a DNS record will be created for a.b.c.ctf - record: "{{ data.name }}" + record: "{{ data.name }}" # Can also be removed or null # This second half of the IPv6 is derived from the hwaddr. ipv6: "{{ data.full_ipv6_address }}" - config: {% if data.is_windows %} - security.secureboot: "false"{% else %}{}{% endif %} - devices: {% if data.is_windows %} + config: {} + devices: {% if data.is_windows or data.with_virtual_machine %} - name: "incusagent" type: "disk" properties: - source: "agent:config"{% else %}[]{% endif %}{% if data.is_windows %} + source: "agent:config"{% else %}[]{% endif %}{% if data.is_windows or data.with_virtual_machine %} wait_for: - type: "agent"{% endif %} + - type: "agent"{% endif %} services: # List of network services running for the track. # Each service name must be unique for a given instance (container/VM). diff --git a/ctf/templates/new/infra-skeleton/deploy.yaml.j2 b/ctf/templates/new/infra-skeleton/deploy.yaml.j2 new file mode 100644 index 0000000..612326f --- /dev/null +++ b/ctf/templates/new/infra-skeleton/deploy.yaml.j2 @@ -0,0 +1,18 @@ +# This is the main ansible script to deploy the challenge. + +# Example on how to run stuff on all hosts of the track +- name: "{{ data.name }} Deployment Playbook" + hosts: all{% if data.with_build %},!build{% endif %} + vars_files: + - ../track.yaml + tasks: + # This is a helper task that loads the tracks' `track.yaml` file and loads the flags as + # ansible facts (like variables) to use in subsequent steps. The key is the `discourse` tag + # of the flag. See the index.php file for an example on how to use/print the flags. + - name: "Load flags" + loop: "{{ '{{ flags }}' }}" + vars: + key: "{{ '{{ (item.tags).discourse }}' }}" + value: "{{ '{{ item.flag }}' }}" + ansible.builtin.set_fact: + track_flags: "{{ '{{ track_flags | default({}) | combine({key: value}) }}' }}" diff --git a/ctf/templates/new/python-service/deploy.yaml.j2 b/ctf/templates/new/python-service/deploy.yaml.j2 index 081f14d..74127e7 100644 --- a/ctf/templates/new/python-service/deploy.yaml.j2 +++ b/ctf/templates/new/python-service/deploy.yaml.j2 @@ -31,9 +31,9 @@ - virtualenv state: present -# Run this only on the `{{ data.name }}` host. Sets up apache2 and php. -- name: Specific tasks for "{{ data.name }}" - hosts: "{{ data.name }}" +# Run this only on the `{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}` host. Sets up apache2 and php. +- name: Specific tasks for "{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}" + hosts: "{% if data.with_virtual_machine %}team-orig-{% endif %}{{ data.name }}" tasks: - name: Create service user ansible.builtin.user: diff --git a/ctf/templates/new/windows-vm/deploy.yaml.j2 b/ctf/templates/new/windows-vm/deploy.yaml.j2 index 5648beb..5a23652 100644 --- a/ctf/templates/new/windows-vm/deploy.yaml.j2 +++ b/ctf/templates/new/windows-vm/deploy.yaml.j2 @@ -2,7 +2,7 @@ # Example on how to run stuff on all hosts of the track - name: "Windows deployment" - hosts: all{% if data.with_build %},!build{% endif %} + hosts: "team-orig-{{ data.name }}" vars_files: - ../track.yaml tasks: diff --git a/pyproject.toml b/pyproject.toml index 37be2ab..21e1e2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "typer==0.24.1", "pydantic", ] -version = "5.0.1" +version = "5.1.0" classifiers = [ "Programming Language :: Python :: 3", "Operating System :: OS Independent",