Update the StorPool Cinder patches
The context for some of them changed in the Cinder master branch, so
`git am` was no longer able to apply them directly. Also, import some
updates to the patches - documentation, release notes, etc - so that
they match exactly what is submitted to the OpenDev Gerrit instance for
merging upstream.
Change-Id: Iacd95c275340faa435a2bf3a83a61e28ab8f8bf3
diff --git a/patches/openstack/cinder/sep-sp-iscsi.patch b/patches/openstack/cinder/sep-sp-iscsi.patch
index 840007d..5571ce5 100644
--- a/patches/openstack/cinder/sep-sp-iscsi.patch
+++ b/patches/openstack/cinder/sep-sp-iscsi.patch
@@ -1,4 +1,4 @@
-From 9d022ada82cb1aa161e360890c4d86fce958aea4 Mon Sep 17 00:00:00 2001
+From 6e24ec90deb5e5977a4654c0e9f7f02e99ddb131 Mon Sep 17 00:00:00 2001
From: Peter Penchev <openstack-dev@storpool.com>
Date: Mon, 12 Mar 2018 12:00:10 +0200
Subject: [PATCH 10/10] Add iSCSI export support to the StorPool driver
@@ -25,39 +25,367 @@
Change-Id: I9de64306e0e6976268df782053b0651dd1cca96f
---
- .../unit/volume/drivers/test_storpool.py | 64 ++-
- cinder/volume/drivers/storpool.py | 374 +++++++++++++++++-
- .../drivers/storpool-volume-driver.rst | 49 ++-
- 3 files changed, 478 insertions(+), 9 deletions(-)
+ .../unit/volume/drivers/test_storpool.py | 435 +++++++++++++++++-
+ cinder/volume/drivers/storpool.py | 374 ++++++++++++++-
+ .../drivers/storpool-volume-driver.rst | 60 ++-
+ .../storpool-iscsi-cefcfe590a07c5c7.yaml | 10 +
+ 4 files changed, 870 insertions(+), 9 deletions(-)
+ create mode 100644 releasenotes/notes/storpool-iscsi-cefcfe590a07c5c7.yaml
diff --git a/cinder/tests/unit/volume/drivers/test_storpool.py b/cinder/tests/unit/volume/drivers/test_storpool.py
-index 95a1ffffd..7e8a17800 100644
+index 95a1ffffd..842790ab4 100644
--- a/cinder/tests/unit/volume/drivers/test_storpool.py
+++ b/cinder/tests/unit/volume/drivers/test_storpool.py
-@@ -32,6 +32,7 @@ fakeStorPool.sptypes = mock.Mock()
+@@ -14,15 +14,25 @@
+ # under the License.
+
+
++from __future__ import annotations
++
++import dataclasses
+ import itertools
+ import re
+ import sys
++from typing import Any, NamedTuple, TYPE_CHECKING # noqa: H301
+ from unittest import mock
+
+ import ddt
+ from oslo_utils import units
+ import six
+
++if TYPE_CHECKING:
++ if sys.version_info >= (3, 11):
++ from typing import Self
++ else:
++ from typing_extensions import Self
++
+
+ fakeStorPool = mock.Mock()
+ fakeStorPool.spopenstack = mock.Mock()
+@@ -32,12 +42,21 @@ fakeStorPool.sptypes = mock.Mock()
sys.modules['storpool'] = fakeStorPool
+from cinder.common import constants
from cinder import exception
++from cinder.tests.unit import fake_constants as fconst
from cinder.tests.unit import test
from cinder.volume import configuration as conf
-@@ -222,7 +223,14 @@ class StorPoolTestCase(test.TestCase):
+ from cinder.volume.drivers import storpool as driver
+
+
++_ISCSI_IQN_OURS = 'beleriand'
++_ISCSI_IQN_OTHER = 'rohan'
++_ISCSI_IQN_THIRD = 'gondor'
++_ISCSI_PAT_OTHER = 'roh*'
++_ISCSI_PAT_BOTH = '*riand roh*'
++_ISCSI_PORTAL_GROUP = 'openstack_pg'
++
+ volume_types = {
+ 1: {},
+ 2: {'storpool_template': 'ssd'},
+@@ -71,6 +90,10 @@ def snapshotName(vtype, vid):
+ return 'os--snap--{t}--{id}'.format(t=vtype, id=vid)
+
+
++def targetName(vid):
++ return 'iqn.2012-11.storpool:{id}'.format(id=vid)
++
++
+ class MockDisk(object):
+ def __init__(self, diskId):
+ self.id = diskId
+@@ -181,6 +204,273 @@ def MockVolumeUpdateDesc(size):
+ return {'size': size}
+
+
++@dataclasses.dataclass(frozen=True)
++class MockIscsiNetwork:
++ """Mock a StorPool IP CIDR network definition (partially)."""
++
++ address: str
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiPortalGroup:
++ """Mock a StorPool iSCSI portal group definition (partially)."""
++
++ name: str
++ networks: list[MockIscsiNetwork]
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiExport:
++ """Mock a StorPool iSCSI exported volume/target definition."""
++
++ portalGroup: str
++ target: str
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiInitiator:
++ """Mock a StorPool iSCSI initiator definition."""
++
++ name: str
++ exports: list[MockIscsiExport]
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiTarget:
++ """Mock a StorPool iSCSI volume-to-target mapping definition."""
++
++ name: str
++ volume: str
++
++
++class IscsiTestCase(NamedTuple):
++ """A single test case for the iSCSI config and export methods."""
++
++ initiator: str | None
++ volume: str | None
++ exported: bool
++ commands_count: int
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiConfig:
++ """Mock the structure returned by the "get current config" query."""
++
++ portalGroups: dict[str, MockIscsiPortalGroup]
++ initiators: dict[str, MockIscsiInitiator]
++ targets: dict[str, MockIscsiTarget]
++
++ @classmethod
++ def build(cls, tcase: IscsiTestCase) -> Self:
++ """Build a test config structure."""
++ initiators = {
++ '0': MockIscsiInitiator(name=_ISCSI_IQN_OTHER, exports=[]),
++ }
++ if tcase.initiator is not None:
++ initiators['1'] = MockIscsiInitiator(
++ name=tcase.initiator,
++ exports=(
++ [
++ MockIscsiExport(
++ portalGroup=_ISCSI_PORTAL_GROUP,
++ target=targetName(tcase.volume),
++ ),
++ ]
++ if tcase.exported
++ else []
++ ),
++ )
++
++ targets = {
++ '0': MockIscsiTarget(
++ name=targetName(fconst.VOLUME2_ID),
++ volume=volumeName(fconst.VOLUME2_ID),
++ ),
++ }
++ if tcase.volume is not None:
++ targets['1'] = MockIscsiTarget(
++ name=targetName(tcase.volume),
++ volume=volumeName(tcase.volume),
++ )
++
++ return cls(
++ portalGroups={
++ '0': MockIscsiPortalGroup(
++ name=_ISCSI_PORTAL_GROUP + '-not',
++ networks=[],
++ ),
++ '1': MockIscsiPortalGroup(
++ name=_ISCSI_PORTAL_GROUP,
++ networks=[
++ MockIscsiNetwork(address="192.0.2.0"),
++ MockIscsiNetwork(address="195.51.100.0"),
++ ],
++ ),
++ },
++ initiators=initiators,
++ targets=targets,
++ )
++
++
++@dataclasses.dataclass(frozen=True)
++class MockIscsiConfigTop:
++ """Mock the top level of the "get the iSCSI configuration" response."""
++
++ iscsi: MockIscsiConfig
++
++
++class MockIscsiAPI:
++ """Mock only the iSCSI-related calls of the StorPool API bindings."""
++
++ _asrt: test.TestCase
++ _configs: list[MockIscsiConfig]
++
++ def __init__(
++ self,
++ configs: list[MockIscsiConfig],
++ asrt: test.TestCase,
++ ) -> None:
++ """Store the reference to the list of iSCSI config objects."""
++ self._asrt = asrt
++ self._configs = configs
++
++ def iSCSIConfig(self) -> MockIscsiConfigTop:
++ """Return the last version of the iSCSI configuration."""
++ return MockIscsiConfigTop(iscsi=self._configs[-1])
++
++ def _handle_export(
++ self,
++ cfg: MockIscsiConfig, cmd: dict[str, Any],
++ ) -> MockIscsiConfig:
++ """Add an export for an initiator."""
++ self._asrt.assertDictEqual(
++ cmd,
++ {
++ 'initiator': _ISCSI_IQN_OURS,
++ 'portalGroup': _ISCSI_PORTAL_GROUP,
++ 'volumeName': volumeName(fconst.VOLUME_ID),
++ },
++ )
++ self._asrt.assertEqual(cfg.initiators['1'].name, cmd['initiator'])
++ self._asrt.assertListEqual(cfg.initiators['1'].exports, [])
++
++ return dataclasses.replace(
++ cfg,
++ initiators={
++ **cfg.initiators,
++ '1': dataclasses.replace(
++ cfg.initiators['1'],
++ exports=[
++ MockIscsiExport(
++ portalGroup=cmd['portalGroup'],
++ target=targetName(fconst.VOLUME_ID),
++ ),
++ ],
++ ),
++ },
++ )
++
++ def _handle_create_initiator(
++ self,
++ cfg: MockIscsiConfig,
++ cmd: dict[str, Any],
++ ) -> MockIscsiConfig:
++ """Add a whole new initiator."""
++ self._asrt.assertDictEqual(
++ cmd,
++ {
++ 'name': _ISCSI_IQN_OURS,
++ 'username': '',
++ 'secret': '',
++ },
++ )
++ self._asrt.assertNotIn(
++ cmd['name'],
++ [init.name for init in cfg.initiators.values()],
++ )
++ self._asrt.assertListEqual(sorted(cfg.initiators), ['0'])
++
++ return dataclasses.replace(
++ cfg,
++ initiators={
++ **cfg.initiators,
++ '1': MockIscsiInitiator(name=cmd['name'], exports=[]),
++ },
++ )
++
++ def _handle_create_target(
++ self,
++ cfg: MockIscsiConfig,
++ cmd: dict[str, Any],
++ ) -> MockIscsiConfig:
++ """Add a target for a volume so that it may be exported."""
++ self._asrt.assertDictEqual(
++ cmd,
++ {'volumeName': volumeName(fconst.VOLUME_ID)},
++ )
++ self._asrt.assertListEqual(sorted(cfg.targets), ['0'])
++ return dataclasses.replace(
++ cfg,
++ targets={
++ **cfg.targets,
++ '1': MockIscsiTarget(
++ name=targetName(fconst.VOLUME_ID),
++ volume=volumeName(fconst.VOLUME_ID),
++ ),
++ },
++ )
++
++ def _handle_initiator_add_network(
++ self,
++ cfg: MockIscsiConfig,
++ cmd: dict[str, Any],
++ ) -> MockIscsiConfig:
++ """Add a network that an initiator is allowed to log in from."""
++ self._asrt.assertDictEqual(
++ cmd,
++ {
++ 'initiator': _ISCSI_IQN_OURS,
++ 'net': '0.0.0.0/0',
++ },
++ )
++ return dataclasses.replace(cfg)
++
++ _CMD_HANDLERS = {
++ 'createInitiator': _handle_create_initiator,
++ 'createTarget': _handle_create_target,
++ 'export': _handle_export,
++ 'initiatorAddNetwork': _handle_initiator_add_network,
++ }
++
++ def iSCSIConfigChange(
++ self,
++ commands: dict[str, list[dict[str, dict[str, Any]]]],
++ ) -> None:
++ """Apply the requested changes to the iSCSI configuration.
++
++ This method adds a new config object to the configs list,
++ making a shallow copy of the last one and applying the changes
++ specified in the list of commands.
++ """
++ self._asrt.assertListEqual(sorted(commands), ['commands'])
++ self._asrt.assertGreater(len(commands['commands']), 0)
++ for cmd in commands['commands']:
++ keys = sorted(cmd.keys())
++ cmd_name = keys[0]
++ self._asrt.assertListEqual(keys, [cmd_name])
++ handler = self._CMD_HANDLERS[cmd_name]
++ new_cfg = handler(self, self._configs[-1], cmd[cmd_name])
++ self._configs.append(new_cfg)
++
++
++_ISCSI_TEST_CASES = [
++ IscsiTestCase(None, None, False, 4),
++ IscsiTestCase(_ISCSI_IQN_OURS, None, False, 2),
++ IscsiTestCase(_ISCSI_IQN_OURS, fconst.VOLUME_ID, False, 1),
++ IscsiTestCase(_ISCSI_IQN_OURS, fconst.VOLUME_ID, True, 0),
++]
++
++
+ def MockSPConfig(section = 's01'):
+ res = {}
+ m = re.match('^s0*([A-Za-z0-9]+)$', section)
+@@ -222,7 +512,15 @@ class StorPoolTestCase(test.TestCase):
self.cfg.volume_backend_name = 'storpool_test'
self.cfg.storpool_template = None
self.cfg.storpool_replication = 3
+ self.cfg.iscsi_cinder_volume = False
+ self.cfg.iscsi_export_to = ''
-+ self.cfg.iscsi_portal_group = 'test-group'
-
-+ self._setup_test_driver()
++ self.cfg.iscsi_learn_initiator_iqns = True
++ self.cfg.iscsi_portal_group = _ISCSI_PORTAL_GROUP
+
++ self._setup_test_driver()
+
+ def _setup_test_driver(self):
+ """Initialize a StorPool driver as per the current configuration."""
mock_exec = mock.Mock()
mock_exec.return_value = ('', '')
-@@ -231,7 +239,7 @@ class StorPoolTestCase(test.TestCase):
+@@ -231,7 +529,7 @@ class StorPoolTestCase(test.TestCase):
self.driver.check_for_setup_error()
@ddt.data(
@@ -66,7 +394,7 @@
({'no-host': None}, KeyError),
({'host': 'sbad'}, driver.StorPoolConfigurationInvalid),
({'host': 's01'}, None),
-@@ -247,7 +255,7 @@ class StorPoolTestCase(test.TestCase):
+@@ -247,7 +545,7 @@ class StorPoolTestCase(test.TestCase):
conn)
@ddt.data(
@@ -75,27 +403,27 @@
({'no-host': None}, KeyError),
({'host': 'sbad'}, driver.StorPoolConfigurationInvalid),
)
-@@ -644,3 +652,55 @@ class StorPoolTestCase(test.TestCase):
+@@ -644,3 +942,136 @@ class StorPoolTestCase(test.TestCase):
self.driver.get_pool({
'volume_type': volume_type
}))
+
+ @ddt.data(
+ # The default values
-+ ('', False, constants.STORPOOL, 'beleriand', False),
++ ('', False, constants.STORPOOL, _ISCSI_IQN_OURS, False),
+
+ # Export to all
-+ ('*', True, constants.ISCSI, 'beleriand', True),
-+ ('*', True, constants.ISCSI, 'beleriand', True),
++ ('*', True, constants.ISCSI, _ISCSI_IQN_OURS, True),
++ ('*', True, constants.ISCSI, _ISCSI_IQN_OURS, True),
+
+ # Only export to the controller
-+ ('', False, constants.STORPOOL, 'beleriand', False),
++ ('', False, constants.STORPOOL, _ISCSI_IQN_OURS, False),
+
+ # Some of the not-fully-supported pattern lists
-+ ('roh*', False, constants.STORPOOL, 'beleriand', False),
-+ ('roh*', False, constants.STORPOOL, 'rohan', True),
-+ ('*riand roh*', False, constants.STORPOOL, 'beleriand', True),
-+ ('*riand roh*', False, constants.STORPOOL, 'rohan', True),
++ (_ISCSI_PAT_OTHER, False, constants.STORPOOL, _ISCSI_IQN_OURS, False),
++ (_ISCSI_PAT_OTHER, False, constants.STORPOOL, _ISCSI_IQN_OTHER, True),
++ (_ISCSI_PAT_BOTH, False, constants.STORPOOL, _ISCSI_IQN_OURS, True),
++ (_ISCSI_PAT_BOTH, False, constants.STORPOOL, _ISCSI_IQN_OTHER, True),
+ )
+ @ddt.unpack
+ def test_wants_iscsi(self, iscsi_export_to, use_iscsi, storage_protocol,
@@ -114,7 +442,7 @@
+ def check(conn, forced, expected):
+ """Pass partially or completely valid connector info."""
+ for initiator in (None, hostname):
-+ for host in (None, 'gondor'):
++ for host in (None, _ISCSI_IQN_THIRD):
+ self.assertEqual(
+ self.driver._connector_wants_iscsi({
+ "host": host,
@@ -131,8 +459,89 @@
+ # look at the specified expected value.
+ check({"storpool_wants_iscsi": False}, use_iscsi, expected)
+ check({}, use_iscsi, expected)
++
++ def _validate_iscsi_config(
++ self,
++ cfg: MockIscsiConfig,
++ res: dict[str, Any],
++ tcase: IscsiTestCase,
++ ) -> None:
++ """Make sure the returned structure makes sense."""
++ initiator = res['initiator']
++ cfg_initiator = cfg.initiators.get('1')
++
++ self.assertIs(res['cfg'].iscsi, cfg)
++ self.assertEqual(res['pg'].name, _ISCSI_PORTAL_GROUP)
++
++ if tcase.initiator is None:
++ self.assertIsNone(initiator)
++ else:
++ self.assertIsNotNone(initiator)
++ self.assertEqual(initiator, cfg_initiator)
++
++ if tcase.volume is None:
++ self.assertIsNone(res['target'])
++ else:
++ self.assertIsNotNone(res['target'])
++ self.assertEqual(res['target'], cfg.targets.get('1'))
++
++ if tcase.initiator is None:
++ self.assertIsNone(cfg_initiator)
++ self.assertIsNone(res['export'])
++ else:
++ self.assertIsNotNone(cfg_initiator)
++ if tcase.exported:
++ self.assertIsNotNone(res['export'])
++ self.assertEqual(res['export'], cfg_initiator.exports[0])
++ else:
++ self.assertIsNone(res['export'])
++
++ @ddt.data(*_ISCSI_TEST_CASES)
++ def test_iscsi_get_config(self, tcase: IscsiTestCase) -> None:
++ """Make sure the StorPool iSCSI configuration is parsed correctly."""
++ cfg_orig = MockIscsiConfig.build(tcase)
++ configs = [cfg_orig]
++ iapi = MockIscsiAPI(configs, self)
++ with mock.patch.object(self.driver._attach, 'api', new=lambda: iapi):
++ res = self.driver._get_iscsi_config(
++ _ISCSI_IQN_OURS,
++ fconst.VOLUME_ID,
++ )
++
++ self._validate_iscsi_config(cfg_orig, res, tcase)
++
++ @ddt.data(*_ISCSI_TEST_CASES)
++ def test_iscsi_create_export(self, tcase: IscsiTestCase) -> None:
++ """Make sure _create_iscsi_export() makes the right API calls."""
++ cfg_orig = MockIscsiConfig.build(tcase)
++ configs = [cfg_orig]
++ iapi = MockIscsiAPI(configs, self)
++ with mock.patch.object(self.driver._attach, 'api', new=lambda: iapi):
++ self.driver._create_iscsi_export(
++ {
++ 'id': fconst.VOLUME_ID,
++ 'display_name': fconst.VOLUME_NAME,
++ },
++ {
++ # Yeah, okay, so we cheat a little bit here...
++ 'host': _ISCSI_IQN_OURS + '.hostname',
++ 'initiator': _ISCSI_IQN_OURS,
++ },
++ )
++
++ self.assertEqual(len(configs), tcase.commands_count + 1)
++ cfg_final = configs[-1]
++ self.assertEqual(cfg_final.initiators['1'].name, _ISCSI_IQN_OURS)
++ self.assertEqual(
++ cfg_final.initiators['1'].exports[0].target,
++ targetName(fconst.VOLUME_ID),
++ )
++ self.assertEqual(
++ cfg_final.targets['1'].volume,
++ volumeName(fconst.VOLUME_ID),
++ )
diff --git a/cinder/volume/drivers/storpool.py b/cinder/volume/drivers/storpool.py
-index 58b64bced..f238dc217 100644
+index b15a201c3..ba5aa10c3 100644
--- a/cinder/volume/drivers/storpool.py
+++ b/cinder/volume/drivers/storpool.py
@@ -15,6 +15,7 @@
@@ -581,7 +990,7 @@
'clone_across_pools': True,
'sparse_copy_volume': True,
diff --git a/doc/source/configuration/block-storage/drivers/storpool-volume-driver.rst b/doc/source/configuration/block-storage/drivers/storpool-volume-driver.rst
-index d2c5895a9..c891675bc 100644
+index d2c5895a9..1f3d46cce 100644
--- a/doc/source/configuration/block-storage/drivers/storpool-volume-driver.rst
+++ b/doc/source/configuration/block-storage/drivers/storpool-volume-driver.rst
@@ -19,12 +19,15 @@ Prerequisites
@@ -634,10 +1043,14 @@
Configuring the StorPool volume driver
--------------------------------------
-@@ -55,6 +81,21 @@ volume backend definition) and per volume type:
+@@ -55,6 +81,32 @@ volume backend definition) and per volume type:
with the default placement constraints for the StorPool cluster.
The default value for the chain replication is 3.
++In addition, if the iSCSI protocol is used to access the StorPool cluster as
++described in the previous section, the following options may be defined in
++the ``cinder.conf`` volume backend definition:
++
+- ``iscsi_export_to``: if set to the value ``*``, the StorPool Cinder driver
+ will export volumes and snapshots using the iSCSI protocol instead of
+ the StorPool network protocol. The ``iscsi_portal_group`` option must also
@@ -653,9 +1066,32 @@
+ to attach the volumes and snapshots for transferring data to and from
+ Glance images.
+
++- ``iscsi_learn_initiator_iqns``: if enabled, the StorPool Cinder driver will
++ automatically use the StorPool API to create definitions for new initiators
++ in the StorPool cluster's configuration. This is the default behavior of
++ the driver; it may be disabled in the rare case if, e.g. because of site
++ policy, OpenStack iSCSI initiators (e.g. Nova hypervisors) need to be
++ explicitly allowed to use the StorPool iSCSI targets.
++
Using the StorPool volume driver
--------------------------------
+diff --git a/releasenotes/notes/storpool-iscsi-cefcfe590a07c5c7.yaml b/releasenotes/notes/storpool-iscsi-cefcfe590a07c5c7.yaml
+new file mode 100644
+index 000000000..c48686abb
+--- /dev/null
++++ b/releasenotes/notes/storpool-iscsi-cefcfe590a07c5c7.yaml
+@@ -0,0 +1,10 @@
++---
++features:
++ - |
++ StorPool driver: Added support for exporting the StorPool-backed volumes
++ using the iSCSI protocol, so that the Cinder volume service and/or
++ the Nova or Glance consumers do not need to have the StorPool block
++ device third-party service installed. See the StorPool driver section in
++ the Cinder documentation for more information on the ``iscsi_export_to``,
++ ``iscsi_portal_group``, ``iscsi_cinder_volume``, and
++ ``iscsi_learn_initiator_iqns`` options.
--
-2.39.2
+2.40.1