diff --git a/Fix-the-hardcoding-of-user-role-using-sdk_adapter-ap.patch b/Fix-the-hardcoding-of-user-role-using-sdk_adapter-ap.patch new file mode 100644 index 0000000..b38dc89 --- /dev/null +++ b/Fix-the-hardcoding-of-user-role-using-sdk_adapter-ap.patch @@ -0,0 +1,388 @@ +From 71df0a289d78019dc96168da719c940c9d62e735 Mon Sep 17 00:00:00 2001 +From: chenke +Date: Tue, 17 Sep 2019 15:42:12 +0800 +Subject: [PATCH] Fix the hardcoding of user role using sdk_adapter approach + +1. Refer to nova's latest code, use get_sdk to create the client[1]. + +2. Move placement_client define from default to seperate placement file. + +Note: +The new method for get nova and placement client has been tested in latest +cyborg devstack environment. And the test results are good. + +Because now cyborg does not have unit tests for the common and conf directory, +unit tests related to this patch will be added in the future, but not in this patch. + +Reference: +[1]. https://github.com/openstack/nova/blob/3f019f2db373d82cbc93f0a30de0059d3d07432b/nova/utils.py#L1013 + +Change-Id: I2a335d47ee3cc24f78b4d49942f1cccd586a0dcc +--- + cyborg/common/exception.py | 4 +++ + cyborg/common/nova_client.py | 11 ++------ + cyborg/common/placement_client.py | 15 ++--------- + cyborg/common/utils.py | 45 ++++++++++++++++++++++++++++++- + cyborg/conf/__init__.py | 5 +++- + cyborg/conf/default.py | 43 ----------------------------- + cyborg/conf/nova.py | 42 +++++++++++++++++++++++++++++ + cyborg/conf/placement.py | 42 +++++++++++++++++++++++++++++ + 8 files changed, 140 insertions(+), 67 deletions(-) + create mode 100644 cyborg/conf/nova.py + create mode 100644 cyborg/conf/placement.py + +diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py +index 7c05a3f..318e323 100644 +--- a/cyborg/common/exception.py ++++ b/cyborg/common/exception.py +@@ -196,6 +196,10 @@ class NotFound(CyborgException): + code = http_client.NOT_FOUND + + ++class ServiceUnavailable(Invalid): ++ msg_fmt = _("Service is unavailable at this time.") ++ ++ + class ServiceNotFound(NotFound): + msg_fmt = _("Service %(service_id)s could not be found.") + +diff --git a/cyborg/common/nova_client.py b/cyborg/common/nova_client.py +index 79f4ada..6fa38cf 100644 +--- a/cyborg/common/nova_client.py ++++ b/cyborg/common/nova_client.py +@@ -12,8 +12,7 @@ + # License for the specific language governing permissions and limitations + # under the License. + +-from cyborg.conf import CONF +-from openstack import connection ++from cyborg.common import utils + from oslo_log import log as logging + + LOG = logging.getLogger(__name__) +@@ -21,13 +20,7 @@ LOG = logging.getLogger(__name__) + + class NovaAPI(object): + def __init__(self): +- default_user = "devstack-admin" +- try: +- auth_user = CONF.compute.username +- except Exception: +- auth_user = default_user +- self.conn = connection.Connection(cloud=auth_user) +- self.nova_client = self.conn.compute ++ self.nova_client = utils.get_sdk_adapter('compute') + + def _get_acc_changed_event(self, instance_uuid, dev_profile_name, status): + return [{'name': 'accelerator-requests-bound', +diff --git a/cyborg/common/placement_client.py b/cyborg/common/placement_client.py +index 0f90762..d83bcd6 100644 +--- a/cyborg/common/placement_client.py ++++ b/cyborg/common/placement_client.py +@@ -14,33 +14,22 @@ + # under the License. + + from cyborg.common import exception +-from cyborg.conf import CONF ++from cyborg.common import utils + from keystoneauth1 import exceptions as ks_exc + from oslo_log import log as logging + from oslo_middleware import request_id + +-from openstack import connection +- + + LOG = logging.getLogger(__name__) + NESTED_PROVIDER_API_VERSION = '1.14' + POST_RPS_RETURNS_PAYLOAD_API_VERSION = '1.20' + PLACEMENT_CLIENT_SEMAPHORE = 'placement_client' +-_CONN = None + + + class PlacementClient(object): + """Client class for reporting to placement.""" + def __init__(self): +- global _CONN +- if _CONN is None: +- default_user = 'devstack-admin' +- try: +- auth_user = CONF.placement.username or default_user +- except Exception: +- auth_user = default_user +- _CONN = connection.Connection(cloud=auth_user) +- self._client = _CONN.placement ++ self._client = utils.get_sdk_adapter('placement') + + def get(self, url, version=None, global_request_id=None): + return self._client.get(url, microversion=version, +diff --git a/cyborg/common/utils.py b/cyborg/common/utils.py +index c60e3cf..96300a8 100644 +--- a/cyborg/common/utils.py ++++ b/cyborg/common/utils.py +@@ -19,17 +19,19 @@ import six + + from keystoneauth1 import exceptions as ks_exc + from keystoneauth1 import loading as ks_loading ++from openstack import connection ++from openstack import exceptions as sdk_exc + from os_service_types import service_types + from oslo_concurrency import lockutils + from oslo_log import log + + from cyborg.common import exception ++from cyborg.common.i18n import _ + import cyborg.conf + + + LOG = log.getLogger(__name__) + +- + synchronized = lockutils.synchronized_with_prefix('cyborg-') + _SERVICE_TYPES = service_types.ServiceTypes() + CONF = cyborg.conf.CONF +@@ -111,6 +113,47 @@ def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None, + min_version=min_version, max_version=max_version) + + ++def _get_conf_group(service_type): ++ # Get the conf group corresponding to the service type. ++ confgrp = _SERVICE_TYPES.get_project_name(service_type) ++ if not confgrp or not hasattr(CONF, confgrp): ++ raise exception.ConfGroupForServiceTypeNotFound(stype=service_type) ++ return confgrp ++ ++ ++def _get_auth_and_session(confgrp): ++ ksa_auth = ks_loading.load_auth_from_conf_options(CONF, confgrp) ++ return ks_loading.load_session_from_conf_options( ++ CONF, confgrp, auth=ksa_auth) ++ ++ ++def get_sdk_adapter(service_type, check_service=False): ++ """Construct an openstacksdk-brokered Adapter for a given service type. ++ We expect to find a conf group whose name corresponds to the service_type's ++ project according to the service-types-authority. That conf group must ++ provide ksa auth, session, and adapter options. ++ :param service_type: String name of the service type for which the Adapter ++ is to be constructed. ++ :param check_service: If True, we will query the endpoint to make sure the ++ service is alive, raising ServiceUnavailable if it is not. ++ :return: An openstack.proxy.Proxy object for the specified service_type. ++ :raise: ConfGroupForServiceTypeNotFound If no conf group name could be ++ found for the specified service_type. ++ :raise: ServiceUnavailable if check_service is True and the service is down ++ """ ++ confgrp = _get_conf_group(service_type) ++ sess = _get_auth_and_session(confgrp) ++ try: ++ conn = connection.Connection( ++ session=sess, oslo_conf=CONF, service_types={service_type}, ++ strict_proxies=check_service) ++ except sdk_exc.ServiceDiscoveryException as e: ++ raise exception.ServiceUnavailable( ++ _("The %(service_type)s service is unavailable: %(error)s") % ++ {'service_type': service_type, 'error': six.text_type(e)}) ++ return getattr(conn, service_type) ++ ++ + def get_endpoint(ksa_adapter): + """Get the endpoint URL represented by a keystoneauth1 Adapter. + +diff --git a/cyborg/conf/__init__.py b/cyborg/conf/__init__.py +index 3a7876c..e1a5889 100644 +--- a/cyborg/conf/__init__.py ++++ b/cyborg/conf/__init__.py +@@ -21,6 +21,8 @@ from cyborg.conf import database + from cyborg.conf import default + from cyborg.conf import glance + from cyborg.conf import keystone ++from cyborg.conf import nova ++from cyborg.conf import placement + from cyborg.conf import service_token + + CONF = cfg.CONF +@@ -29,7 +31,8 @@ api.register_opts(CONF) + agent.register_opts(CONF) + database.register_opts(CONF) + default.register_opts(CONF) +-default.register_placement_opts(CONF) + service_token.register_opts(CONF) + glance.register_opts(CONF) + keystone.register_opts(CONF) ++nova.register_opts(CONF) ++placement.register_opts(CONF) +diff --git a/cyborg/conf/default.py b/cyborg/conf/default.py +index 433d52e..73ab305 100644 +--- a/cyborg/conf/default.py ++++ b/cyborg/conf/default.py +@@ -18,11 +18,9 @@ + import os + import socket + +-from keystoneauth1 import loading as k_loading + from oslo_config import cfg + + from cyborg.common.i18n import _ +-from cyborg.conf import utils as confutils + + + exc_log_opts = [ +@@ -65,33 +63,6 @@ path_opts = [ + help=_("Top-level directory for maintaining cyborg's state.")), + ] + +-PLACEMENT_CONF_SECTION = 'placement' +-DEFAULT_SERVICE_TYPE = 'placement' +- +-placement_group = cfg.OptGroup( +- PLACEMENT_CONF_SECTION, +- title='Placement Service Options', +- help="Configuration options for connecting to the placement API service") +- +-placement_opts = [ +- cfg.StrOpt('endpoint_type', +- default='public', +- choices=['public', 'admin', 'internal'], +- help=_('Type of the placement endpoint to use. This endpoint ' +- 'will be looked up in the keystone catalog and should ' +- 'be one of public, internal or admin.')), +- cfg.BoolOpt( +- 'randomize_allocation_candidates', +- default=False, +- help=_('If True, when limiting allocation candidate results, the ' +- 'results will be a random sampling of the full result set. ' +- 'If False, allocation candidates are returned in a ' +- 'deterministic but undefined order. That is, all things ' +- 'being equal, two requests for allocation candidates will ' +- 'return the same results in the same order; but no guarantees ' +- 'are made as to how that order is determined.')), +-] +- + + def register_opts(conf): + conf.register_opts(exc_log_opts) +@@ -99,24 +70,10 @@ def register_opts(conf): + conf.register_opts(path_opts) + + +-def register_placement_opts(cfg=cfg.CONF): +- cfg.register_group(placement_group) +- cfg.register_opts(placement_opts, group=PLACEMENT_CONF_SECTION) +- confutils.register_ksa_opts(cfg, placement_group, DEFAULT_SERVICE_TYPE) +- +- + DEFAULT_OPTS = (exc_log_opts + service_opts + path_opts) + + + def list_opts(): + return { +- PLACEMENT_CONF_SECTION: ( +- placement_opts + +- k_loading.get_session_conf_options() + +- k_loading.get_auth_common_conf_options() + +- k_loading.get_auth_plugin_conf_options('password') + +- k_loading.get_auth_plugin_conf_options('v2password') + +- k_loading.get_auth_plugin_conf_options('v3password') + +- confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)), + 'DEFAULT': DEFAULT_OPTS + } +diff --git a/cyborg/conf/nova.py b/cyborg/conf/nova.py +new file mode 100644 +index 0000000..edd5dab +--- /dev/null ++++ b/cyborg/conf/nova.py +@@ -0,0 +1,42 @@ ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from keystoneauth1 import loading as ks_loading ++from oslo_config import cfg ++ ++from cyborg.conf import utils as confutils ++ ++ ++DEFAULT_SERVICE_TYPE = 'compute' ++ ++ ++nova_group = cfg.OptGroup( ++ 'nova', ++ title='Nova Service Options', ++ help="Configuration options for connecting to the Nova API service") ++ ++ ++def register_opts(conf): ++ conf.register_group(nova_group) ++ confutils.register_ksa_opts(conf, nova_group, DEFAULT_SERVICE_TYPE) ++ ++ ++def list_opts(): ++ return { ++ nova_group.name: ( ++ ks_loading.get_session_conf_options() + ++ ks_loading.get_auth_common_conf_options() + ++ ks_loading.get_auth_plugin_conf_options('password') + ++ ks_loading.get_auth_plugin_conf_options('v2password') + ++ ks_loading.get_auth_plugin_conf_options('v3password') + ++ confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)) ++ } +diff --git a/cyborg/conf/placement.py b/cyborg/conf/placement.py +new file mode 100644 +index 0000000..cf63314 +--- /dev/null ++++ b/cyborg/conf/placement.py +@@ -0,0 +1,42 @@ ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from keystoneauth1 import loading as ks_loading ++from oslo_config import cfg ++ ++from cyborg.conf import utils as confutils ++ ++ ++PLACEMENT_CONF_SECTION = 'placement' ++DEFAULT_SERVICE_TYPE = 'placement' ++ ++placement_group = cfg.OptGroup( ++ PLACEMENT_CONF_SECTION, ++ title='Placement Service Options', ++ help="Configuration options for connecting to the placement API service") ++ ++ ++def register_opts(conf): ++ conf.register_group(placement_group) ++ confutils.register_ksa_opts(conf, placement_group, DEFAULT_SERVICE_TYPE) ++ ++ ++def list_opts(): ++ return { ++ PLACEMENT_CONF_SECTION: ( ++ ks_loading.get_session_conf_options() + ++ ks_loading.get_auth_common_conf_options() + ++ ks_loading.get_auth_plugin_conf_options('password') + ++ ks_loading.get_auth_plugin_conf_options('v2password') + ++ ks_loading.get_auth_plugin_conf_options('v3password') + ++ confutils.get_ksa_adapter_opts(DEFAULT_SERVICE_TYPE)) ++ } + +-- +2.23.0.windows.1 + diff --git a/openstack-cyborg-agent.service b/openstack-cyborg-agent.service new file mode 100644 index 0000000..5591a78 --- /dev/null +++ b/openstack-cyborg-agent.service @@ -0,0 +1,14 @@ +[Unit] +Description=OpenStack Cyborg Agent Server +After=syslog.target network.target + +[Service] +Type=notify +NotifyAccess=all +TimeoutStartSec=0 +Restart=always +User=cyborg +ExecStart=/usr/bin/cyborg-agent --config-file=/etc/cyborg/cyborg.conf + +[Install] +WantedBy=multi-user.target diff --git a/openstack-cyborg-api.service b/openstack-cyborg-api.service new file mode 100644 index 0000000..3f06f02 --- /dev/null +++ b/openstack-cyborg-api.service @@ -0,0 +1,14 @@ +[Unit] +Description=OpenStack Cyborg API Server +After=syslog.target network.target + +[Service] +Type=notify +NotifyAccess=all +TimeoutStartSec=0 +Restart=always +User=cyborg +ExecStart=/usr/bin/cyborg-api --config-file=/etc/cyborg/cyborg.conf + +[Install] +WantedBy=multi-user.target diff --git a/openstack-cyborg-conductor.service b/openstack-cyborg-conductor.service new file mode 100644 index 0000000..181728e --- /dev/null +++ b/openstack-cyborg-conductor.service @@ -0,0 +1,14 @@ +[Unit] +Description=OpenStack Cyborg Conductor Server +After=syslog.target network.target + +[Service] +Type=notify +NotifyAccess=all +TimeoutStartSec=0 +Restart=always +User=cyborg +ExecStart=/usr/bin/cyborg-conductor --config-file=/etc/cyborg/cyborg.conf + +[Install] +WantedBy=multi-user.target diff --git a/openstack-cyborg.spec b/openstack-cyborg.spec index f12a9e1..55c2b91 100644 --- a/openstack-cyborg.spec +++ b/openstack-cyborg.spec @@ -1,11 +1,17 @@ Name: openstack-cyborg Version: 3.0.1 -Release: 2 +Release: 3 Summary: OpenStack Accelerator as a Service License: Apache-2.0 Group: Applications/System URL: https://launchpad.net/cyborg Source0: https://tarballs.opendev.org/openstack/cyborg/openstack-cyborg-3.0.1.tar.gz +Source1: openstack-cyborg-api.service +Source2: openstack-cyborg-conductor.service +Source3: openstack-cyborg-agent.service + +Patch0000: Fix-the-hardcoding-of-user-role-using-sdk_adapter-ap.patch + BuildRequires: fdupes BuildRequires: openstack-macros BuildRequires: openstack-tempest @@ -29,7 +35,9 @@ BuildRequires: python3-oslotest BuildRequires: python3-pecan BuildRequires: python3-psutil BuildRequires: python3-six -Requires: python3-cyborg = %{version}-%{release} +Requires: openstack-cyborg-api = %{version}-%{release} +Requires: openstack-cyborg-conductor = %{version}-%{release} +Requires: openstack-cyborg-agent = %{version}-%{release} BuildArch: noarch BuildRequires: systemd @@ -45,9 +53,36 @@ provide a general purpose management framework for acceleration resources (i.e. various types of accelerators such as GPU, FPGA, ASIC, NP, SoCs, NVMe/NOF SSDs, ODP, DPDK/SPDK and so on). +%package api +Summary: Cyborg API service +Requires: openstack-cyborg-common = %{version}-%{release} + +%description api +Cyborg API service + +%package conductor +Summary: Cyborg conductor service +Requires: openstack-cyborg-common = %{version}-%{release} + +%description conductor +Cyborg conductor service + +%package agent +Summary: Cyborg agent service +Requires: openstack-cyborg-common = %{version}-%{release} + +%description agent +Cyborg agent service + +%package common +Summary: Cyborg common files +Requires: python3-cyborg = %{version}-%{release} + +%description common +Cyborg common files + %package -n python3-cyborg Summary: Cyborg Python library -Group: Applications/System Requires: python3-sqlalchemy Requires: python3-wsme Requires: python3-alembic @@ -102,25 +137,61 @@ mv %{buildroot}%{_prefix}%{_sysconfdir}/cyborg/* %{buildroot}%{_sysconfdir}/cyb install -p -D -m 640 etc/cyborg/cyborg.conf.sample %{buildroot}%{_sysconfdir}/cyborg/cyborg.conf install -p -D -m 640 etc/cyborg/policy.json.sample %{buildroot}%{_sysconfdir}/cyborg/policy.json +install -p -D -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/openstack-cyborg-api.service +install -p -D -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/openstack-cyborg-conductor.service +install -p -D -m 644 %{SOURCE3} %{buildroot}%{_unitdir}/openstack-cyborg-agent.service + # remove duplicates %fdupes %{buildroot}%{python3_sitelib} -%pre +%pre common %openstack_pre_user_group_create cyborg cyborg /sbin/nologin +%post api +%systemd_post openstack-cyborg-api.service +%post conductor +%systemd_post openstack-cyborg-conductor.service +%post agent +%systemd_post openstack-cyborg-agent.service + +%preun api +%systemd_preun openstack-cyborg-api.service +%preun conductor +%systemd_preun openstack-cyborg-conductor.service +%preun agent +%systemd_preun openstack-cyborg-agent.service + +%postun api +%systemd_postun_with_restart openstack-cyborg-api.service +%postun conductor +%systemd_postun_with_restart openstack-cyborg-conductor.service +%postun agent +%systemd_postun_with_restart openstack-cyborg-agent.service + %files %license LICENSE %doc README.rst + +%files api %{_bindir}/cyborg-api -%{_bindir}/cyborg-conductor %{_bindir}/cyborg-dbsync -%{_bindir}/cyborg-agent %{_bindir}/cyborg-status %{_bindir}/cyborg-wsgi-api -%dir %attr(0750, root, cyborg) %{_sysconfdir}/cyborg/ -%config(noreplace) %attr(0640, root, cyborg) %{_sysconfdir}/cyborg/cyborg.conf %config(noreplace) %attr(0640, root, cyborg) %{_sysconfdir}/cyborg/api-paste.ini %config(noreplace) %attr(0640, root, cyborg) %{_sysconfdir}/cyborg/policy.json +%{_unitdir}/openstack-cyborg-api.service + +%files conductor +%{_bindir}/cyborg-conductor +%{_unitdir}/openstack-cyborg-conductor.service + +%files agent +%{_bindir}/cyborg-agent +%{_unitdir}/openstack-cyborg-agent.service + +%files common +%dir %attr(0750, root, cyborg) %{_sysconfdir}/cyborg/ +%config(noreplace) %attr(0640, root, cyborg) %{_sysconfdir}/cyborg/cyborg.conf %dir %attr(0750, cyborg, cyborg) %{_localstatedir}/log/cyborg %files -n python3-cyborg @@ -130,5 +201,7 @@ install -p -D -m 640 etc/cyborg/policy.json.sample %{buildroot}%{_sysconfdir}/cy %{python3_sitelib}/openstack_cyborg*.egg-info %changelog -* Fri Nov 05 2021 wangxiyuan - 3.0.1-1 +* Sat Nov 27 2021 wangxiyuan - 3.0.1-3 +- Fix cyborg build issue +* Fri Nov 05 2021 wangxiyuan - 3.0.1-2 - Support OpenStack Train release