Add a Kolla Ansible-based Tempest/OpenStack setup

Add a setup that runs a Tempest suite of tests against an OpenStack
deployed by Kolla Ansible.

The setup is similar to the iSCSI multipath one.

Change-Id: Ib41aa7f3a7110ef3afc20076d54d01c473e1e827
diff --git a/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/globals.yml.j2 b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/globals.yml.j2
new file mode 100644
index 0000000..fc0c678
--- /dev/null
+++ b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/globals.yml.j2
@@ -0,0 +1,14 @@
+kolla_base_distro: "{{ kolla_kolla_base_distro }}"
+openstack_release: "{{ kolla_openstack_release }}"
+network_interface: "{{ kolla_network_interface }}"
+neutron_external_interface: "{{ kolla_neutron_external_interface }}"
+kolla_internal_vip_address: "{{ kolla_kolla_internal_vip_address }}"
+enable_cinder: "yes"
+enable_cinder_backend_lvm: "yes"
+
+# TODO: Temporary hardcode private variables.
+#       The proper solution is to mirror the upstream images
+#       from quay.io to cts.storpool.com, and configure kolla-ansible
+#       to use cts.storpool.com for all images.
+nova_compute_image_full: "cts.storpool.com/kolla/ubuntu-binary-nova-compute:zed-ubuntu-jammy.20230614.1"
+cinder_volume_image_full: "cts.storpool.com/kolla/ubuntu-binary-cinder-volume:zed-ubuntu-jammy"
diff --git a/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/post.yaml b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/post.yaml
new file mode 100644
index 0000000..297f6fa
--- /dev/null
+++ b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/post.yaml
@@ -0,0 +1,32 @@
+- hosts: undercloud-client
+  vars_files:
+    - vars.yaml
+  tasks:
+    - name: Remove the Kolla Ports from the OpenStack Node
+      ansible.builtin.command:
+        argv:
+          - "{{ os_venv }}/bin/openstack"
+          - --os-cloud
+          - openstack-testing
+          - server
+          - remove
+          - port
+          - "{{ openstack_node }}"
+          - "{{ item }}"
+      loop:
+        - "{{ kolla_network_interface_port }}"
+        - "{{ kolla_neutron_external_interface_port }}"
+
+    - name: Delete the Kolla Ports
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "port", "delete", "{{ item }}" ]
+      loop:
+        - "{{ kolla_network_interface_port }}"
+        - "{{ kolla_neutron_external_interface_port }}"
+
+    - name: Delete the StorPool Networks
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "network", "delete", "{{ item }}" ]
+      loop:
+        - "{{ kolla_network_interface_network }}"
+        - "{{ kolla_neutron_external_interface_network }}"
\ No newline at end of file
diff --git a/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/pre.yaml b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/pre.yaml
new file mode 100644
index 0000000..a06f1de
--- /dev/null
+++ b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/pre.yaml
@@ -0,0 +1,266 @@
+---
+- hosts: undercloud-client
+  vars_files:
+    - vars.yaml
+  tasks:
+    - name: Create the Kolla Networks
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "network", "create", "{{ item }}" ]
+      loop:
+        - "{{ kolla_network_interface_network }}"
+        - "{{ kolla_neutron_external_interface_network }}"
+
+    - name: Create the Kolla Subnets
+      ansible.builtin.command:
+        argv:
+          - "{{ os_venv }}/bin/openstack"
+          - --os-cloud
+          - openstack-testing
+          - subnet
+          - create
+          - --subnet-range
+          - "{{ item.ip_range }}"
+          - --network
+          - "{{ item.network }}"
+          - "{{ item.subnet }}"
+      loop:
+        - { ip_range: "{{ network_ip_kolla_network_interface }}", network: "{{ kolla_network_interface_network }}", subnet: "{{ kolla_network_interface_subnet }}" }
+        - { ip_range: "{{ network_ip_kolla_neutron_external_interface }}", network: "{{ kolla_neutron_external_interface_network }}", subnet: "{{ kolla_neutron_external_interface_subnet }}" }
+
+    # TODO: Support port security
+    - name: Create the Kolla Ports
+      ansible.builtin.command:
+        argv:
+          - "{{ os_venv }}/bin/openstack"
+          - --os-cloud
+          - openstack-testing
+          - port
+          - create
+          - --network
+          - "{{ item.network }}"
+          - --fixed-ip
+          - "subnet={{ item.subnet }},ip-address={{ item.ip }}"
+          - --disable-port-security
+          - "{{ item.port }}"
+      loop:
+        - { network: "{{ kolla_network_interface_network }}", subnet: "{{ kolla_network_interface_subnet }}", ip: "{{ ip_kolla_network_interface }}", port: "{{ kolla_network_interface_port }}" }
+        - { network: "{{ kolla_neutron_external_interface_network }}", subnet: "{{ kolla_neutron_external_interface_subnet }}", ip: "{{ ip_kolla_neutron_external_interface }}", port: "{{ kolla_neutron_external_interface_port }}" }
+
+    - name: Attach the Kolla Ports to the OpenStack Node
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "server", "add", "port", "{{ openstack_node }}", "{{ item }}" ]
+      loop:
+        - "{{ kolla_network_interface_port }}"
+        - "{{ kolla_neutron_external_interface_port }}"
+
+    - name: Get Information About kolla_network_interface_port
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "port", "show", "--format", "json", "{{ kolla_network_interface_port }}" ]
+      register: kolla_network_interface_port_info
+
+    - name: Get MAC Address of kolla_network_interface_port
+      ansible.builtin.set_fact:
+        kolla_network_interface_port_info_mac: "{{ (kolla_network_interface_port_info.stdout | from_json).mac_address }}"
+
+    - name: Get Information About kolla_neutron_external_interface_port
+      ansible.builtin.command:
+        argv: [ "{{ os_venv }}/bin/openstack", "--os-cloud", "openstack-testing", "port", "show", "--format", "json", "{{ kolla_neutron_external_interface_port }}" ]
+      register: kolla_neutron_external_interface_port_info
+
+    - name: Get MAC Address of kolla_neutron_external_interface_port
+      ansible.builtin.set_fact:
+        kolla_neutron_external_interface_port_mac: "{{ (kolla_neutron_external_interface_port_info.stdout | from_json).mac_address }}"
+
+- name: Setup LVM to be a Cinder Backend
+  hosts: controller
+  tasks:
+    - name: Get a Free Device
+      shell: losetup -f
+      become: true
+      register: free_device
+
+    - name: fallocate 8G in a File
+      shell: fallocate -l 32G /var/lib/cinder_data.img
+      become: true
+
+    - name: Setup the loopback file
+      shell: losetup "{{ free_device.stdout }}" /var/lib/cinder_data.img
+      become: true
+
+    - name: Create a Physical Volume
+      shell: pvcreate "{{ free_device.stdout }}"
+      become: true
+
+    - name: Create a Volume Group
+      shell: vgcreate cinder-volumes "{{ free_device.stdout }}"
+      become: true
+
+- name: Install Kolla
+  hosts: controller
+  vars_files:
+    - vars.yaml
+  tasks:
+    - set_fact:
+        iscsi_mac: "{{ hostvars['undercloud-client']['kolla_network_interface_port_info_mac'] }}"
+        iscsi_ip: "{{ ip_kolla_network_interface }}/24"
+
+    - set_fact:
+        ip_kolla_network_kolla: "{{ hostvars['localhost'].ISCSI_NODE_SYSTEMD_NETWORKD.v }}"
+      no_log: true
+
+    - name: Provision the Netplan Template for the Kolla "network_interface"
+      no_log: true
+      become: true
+      ansible.builtin.copy:
+        content: "{{ ip_kolla_network_kolla }}"
+        dest: /etc/systemd/network/70-kolla-network-interface.network
+
+    - name: Restart systemd-networkd to Apply the Network Configuration
+      become: true
+      ansible.builtin.command:
+        argv: [ "systemctl", "restart", "systemd-networkd" ]
+
+    - name: Find the Name of Interface for 'network_interface'
+      set_fact:
+        kolla_network_interface: "{{ item }}"
+      loop: "{{ ansible_interfaces }}"
+      when: ansible_facts[item]['macaddress']|default(None) == hostvars['undercloud-client']['kolla_network_interface_port_info_mac']
+
+    - name: Find the Name of Interface for 'network_interface'
+      set_fact:
+        kolla_neutron_external_interface: "{{ item }}"
+      loop: "{{ ansible_interfaces }}"
+      when: ansible_facts[item]['macaddress']|default(None) == hostvars['undercloud-client']['kolla_neutron_external_interface_port_mac']
+
+    - name: Install Dependencies
+      become: true
+      ansible.builtin.apt:
+        name:
+          - git
+          - python3-dev
+          - python3-pip
+          - libffi-dev
+          - gcc
+          - libssl-dev
+        state: latest
+        update_cache: true
+
+    - name: Install Dependencies for the Virtual Environment
+      become: true
+      ansible.builtin.apt:
+        name:
+          - python3-venv
+        state: latest
+        update_cache: true
+
+    - name: Install virtualenv
+      become: true
+      ansible.builtin.pip:
+        name: virtualenv
+        executable: pip3
+
+    - name: Create a Kolla Venv
+      ansible.builtin.pip:
+        name:
+          - 'ansible>=6,<8'
+          - git+https://opendev.org/openstack/kolla-ansible@master
+          - python-openstackclient
+        virtualenv: "{{ kolla_venv }}"
+
+    - name: Create the /etc/kolla Directory
+      become: true
+      ansible.builtin.file:
+        path: /etc/kolla
+        state: directory
+        mode: '0755'
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+
+    - name: Copy the Password Template File to /etc/kolla
+      ansible.builtin.copy:
+        src: "{{ kolla_venv }}/share/kolla-ansible/etc_examples/kolla/passwords.yml"
+        dest: /etc/kolla/passwords.yml
+        remote_src: true
+
+    - name: Provision the globals.yml Template
+      ansible.builtin.template:
+        src: globals.yml.j2
+        dest: /etc/kolla/globals.yml
+
+    - name: Install Ansible Galaxy Requirements
+      shell: . {{ kolla_venv }}/bin/activate && LC_ALL=en_US.UTF-8 kolla-ansible install-deps
+
+    - name: Generate Kolla Password
+      shell: . {{ kolla_venv }}/bin/activate && kolla-genpwd
+
+    - name: Bootstrap Servers with Kolla Deploy Dependencies
+      shell: . {{ kolla_venv }}/bin/activate && LC_ALL=en_US.UTF-8 kolla-ansible -i {{ kolla_venv}}/share/kolla-ansible/ansible/inventory/all-in-one bootstrap-servers
+
+    - name: Do Pre-Deployment Checks for Hosts
+      shell: . {{ kolla_venv }}/bin/activate && LC_ALL=en_US.UTF-8 kolla-ansible -i {{ kolla_venv}}/share/kolla-ansible/ansible/inventory/all-in-one prechecks
+
+    - name: Finally proceed to actual OpenStack deployment
+      shell: . {{ kolla_venv }}/bin/activate && LC_ALL=en_US.UTF-8 kolla-ansible -i {{ kolla_venv}}/share/kolla-ansible/ansible/inventory/all-in-one deploy
+
+    - name: Generate clouds.yaml
+      shell: . {{ kolla_venv }}/bin/activate && LC_ALL=en_US.UTF-8 kolla-ansible -i {{ kolla_venv}}/share/kolla-ansible/ansible/inventory/all-in-one post-deploy
+
+    - name: Copy clouds.yaml to /etc/openstack
+      ansible.builtin.copy:
+        src: /etc/kolla/clouds.yaml
+        dest: /etc/openstack/clouds.yaml
+        remote_src: true
+
+    - name: Wait for Cluster to Come Up
+      shell: sleep 60
+
+    - name: init-runonce
+      shell: . {{ kolla_venv }}/bin/activate && ./share/kolla-ansible/init-runonce
+      args:
+        chdir: "{{ kolla_venv }}"
+
+    - name: Install openvswitch-switch
+      shell: apt-get install -y openvswitch-switch
+      become: true
+      ignore_errors: true
+
+    - name: Enable Provider Network
+      shell: "{{ item }}"
+      become: true
+      loop:
+        - ip link add veth1 type veth peer name veth2
+        - ip addr add 10.0.2.149/24 dev veth1
+        - ip link set veth1 up
+        - ip link set veth2 up
+        - ovs-vsctl add-port br-ex veth2
+
+- name: Setup Tempest
+  hosts: controller
+  vars_files:
+    - vars.yaml
+  tasks:
+    - name: Create Flavors that Tempest Uses for Testing
+      shell: . {{ kolla_venv }}/bin/activate && {{ item }}
+      loop:
+        # 64MB is not enough for recent Cirros images: https://github.com/cirros-dev/cirros/issues/63
+        #- openstack --os-cloud kolla-admin flavor create --id auto --ram 64 --disk 1 --vcpus 1 m1.nano
+        - openstack --os-cloud kolla-admin flavor create --id auto --ram 128 --disk 1 --vcpus 1 m1.nano
+        - openstack --os-cloud kolla-admin flavor create --id auto --ram 256 --disk 1 --vcpus 1 m1.micro
+
+    - name: Create a Tempest Venv
+      ansible.builtin.pip:
+        name:
+          - git+https://opendev.org/openstack/tempest@34.2.0
+          - python-tempestconf
+        virtualenv: "{{ tempest_venv }}"
+
+    - name: Initialize Tempest Cloud cloud-01
+      shell: . {{ tempest_venv }}/bin/activate && tempest init cloud-01
+      args:
+        chdir: "{{ ansible_env.HOME }}"
+        executable: /bin/bash
+
+    - name: Configure Tempest with the Kolla OpenStack Cloud
+      shell: . {{ tempest_venv }}/bin/activate && discover-tempest-config --debug --os-cloud kolla-admin
+      args:
+        chdir: "{{ ansible_env.HOME }}/cloud-01"
\ No newline at end of file
diff --git a/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/vars.yaml b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/vars.yaml
new file mode 100644
index 0000000..e811410
--- /dev/null
+++ b/playbooks/kolla-setup-openstack-iscsi-multipath-storpool/vars.yaml
@@ -0,0 +1,26 @@
+kolla_venv: "{{ ansible_env.HOME }}/kolla-venv"
+tempest_venv: "{{ ansible_env.HOME }}/tempest-venv"
+
+kolla_kolla_base_distro: "ubuntu"
+kolla_openstack_release: "zed"
+#kolla_network_interface: "enp4s0"
+#kolla_neutron_external_interface: "enp5s0"
+kolla_kolla_internal_vip_address: "192.168.60.250"
+
+undercloud_client_node: "{{ hostvars['undercloud-client']['nodepool']['external_id'] }}"
+openstack_node: "{{ hostvars['controller']['nodepool']['external_id'] }}"
+
+network_ip_kolla_network_interface: "192.168.60.0/24"
+network_ip_kolla_neutron_external_interface: "192.168.70.0/24"
+
+ip_kolla_network_interface: "192.168.60.3"
+ip_kolla_neutron_external_interface: "192.168.70.3"
+
+kolla_network_interface_network: "{{ undercloud_client_node }}-network-kolla-network-interface"
+kolla_neutron_external_interface_network: "{{ undercloud_client_node }}-network-kolla-neutron-external-interface"
+
+kolla_network_interface_subnet: "{{ undercloud_client_node }}-subnet-kolla-network-interface"
+kolla_neutron_external_interface_subnet: "{{ undercloud_client_node }}-subnet-kolla-neutron-external-interface"
+
+kolla_network_interface_port: "{{ undercloud_client_node }}-port-kolla-network-interface"
+kolla_neutron_external_interface_port: "{{ undercloud_client_node }}-port-kolla-neutron-external-interface"
\ No newline at end of file
diff --git a/playbooks/kolla-tempest-run.yaml b/playbooks/kolla-tempest-run.yaml
new file mode 100644
index 0000000..637ba4f
--- /dev/null
+++ b/playbooks/kolla-tempest-run.yaml
@@ -0,0 +1,11 @@
+- name: Run Tempest Against a Kolla OpenStack
+  hosts: controller
+  vars_files:
+    - vars.yaml
+  tasks:
+    - name: Debug Sleep
+      shell: sleep 3600
+#    - name: Run Tempest
+#      shell: . {{ tempest_venv }}/bin/activate && tempest run --regex volume
+#      args:
+#        chdir: "{{ ansible_env.HOME }}/cloud-01"
\ No newline at end of file
diff --git a/zuul.d/sp-cinder.yaml b/zuul.d/sp-cinder.yaml
index 7d4ac75..d0dbb44 100644
--- a/zuul.d/sp-cinder.yaml
+++ b/zuul.d/sp-cinder.yaml
@@ -206,6 +206,55 @@
 #        ^cinder_tempest_plugin
 #        (^(tempest\.((api\..*volume)|scenario\.test_encrypted_cinder_volumes|scenario\.test_volume|scenario\.test_shelve_instance))|(cinder_tempest_plugin))
 
+- job:
+    name: kolla-cinder-storpool-tempest-iscsi-multipath
+    pre-run:
+      - playbooks/setup-openstack-client/pre.yaml
+      - playbooks/setup-openstack-iscsi-multipath/pre.yaml
+      - playbooks/setup-openstack-iscsi-multipath-storpool/pre.yaml
+      - playbooks/kolla-setup-openstack-iscsi-multipath-storpool/pre.yaml
+    run: playbooks/kolla-tempest-run.yaml
+    post-run:
+      - playbooks/kolla-setup-openstack-iscsi-multipath-storpool/post.yaml
+      - playbooks/setup-openstack-iscsi-multipath-storpool/post.yaml
+      - playbooks/setup-openstack-iscsi-multipath/post.yaml
+      - playbooks/setup-openstack-client/post.yaml
+    required-projects:
+      - opendev.org/openstack/tempest
+    timeout: 10800
+    nodeset: openstack-multi-node-mixed
+    attempts: 1
+    secrets:
+      # openstack-client
+      - name: OPENSTACK_DATA
+        secret: OPENSTACK_DATA
+      - name: RESOLVED_DATA
+        secret: RESOLVED_DATA
+
+      # openstack-iscsi-multipath-storpool
+      - name: STORPOOL_DEPLOY_KEY
+        secret: STORPOOL_DEPLOY_KEY
+      - name: STORPOOL_DEPLOY_KEY_PUB
+        secret: STORPOOL_DEPLOY_KEY_PUB
+      - name: STORPOOL_INVENTORY
+        secret: STORPOOL_INVENTORY
+      - name: STORPOOL_NETPLAN
+        secret: STORPOOL_NETPLAN
+      - name: STORPOOL_NETPLAN_SERVICE
+        secret: STORPOOL_NETPLAN_SERVICE
+      - name: STORPOOL_CONF
+        secret: STORPOOL_CONF
+      - name: STORPOOL_CONF_ISCSI_NODE
+        secret: STORPOOL_CONF_ISCSI_NODE
+      - name: ISCSI_NODE_SYSTEMD_NETWORKD
+        secret: ISCSI_NODE_SYSTEMD_NETWORKD
+    vars:
+      os_venv: "~/sp-venv-openstack"
+      zuul_copy_output:
+        /etc/multipath.conf: logs_txt
+        /etc/iscsi/iscsid.conf: logs_txt
+      tempest_test_regex: volume
+
 - project:
     name: openstack/cinder
     check:
@@ -214,12 +263,14 @@
             branches: master
         - cinder-storpool-tempest-iscsi-multipath:
             branches: master
-    #experimental:
-    #  jobs:
-    #    #- cinder-storpool-tempest-experimental:
-    #    #    branches: master
-    #    #- cinder-storpool-tempest-iscsi:
-    #    #    branches: master
+    experimental:
+      jobs:
+        - kolla-cinder-storpool-tempest-iscsi-multipath:
+            branches: master
+        #- cinder-storpool-tempest-experimental:
+        #    branches: master
+        #- cinder-storpool-tempest-iscsi:
+        #    branches: master
 
 - project:
     name: openstack/os-brick