ubs全称是utilization-based scheduler，算是对scheduler filter的一种扩展实现。计算节点通过resource_tracker收集监控的资源，存入数据库的compute_nodes的metric字段。

这里演示根据ceph osd系统盘的数量来优先调度的场景(纯属demo)

环境： rdo L版

1、在nova setup.cfg中增加一条记录（rpm包装出来的是entry_points.txt）

vim/usr/lib/python2.7/site-packages/nova-4.0rc6-py2.7.egg-info/entry_points.txt[nova.compute.monitors.osd]virt_driver=nova.compute.monitors.osd.virt_driver:Monitor#compute.monitor下已经有cpu的实现了

2、加载osd monitor

vim/usr/lib/python2.7/site-packages/nova/compute/monitors/__init__.pyclassMonitorHandler(object):NAMESPACES=['nova.compute.monitors.cpu','nova.compute.monitors.osd',#增加osdmonitor]

3、

vim/usr/lib/python2.7/site-packages/nova/compute/monitors/base.py#增加MonitorBase_OSD类、OSDMonitorBase类@six.add_metaclass(abc.ABCMeta)classMonitorBase_OSD(object):"""Baseclassforallresourcemonitorplugins."""def__init__(self,compute_manager):self.compute_manager=compute_managerself.source=None@abc.abstractmethoddefget_metric(self,name):"""Returna(value,timestamp)tupleforthesuppliedmetricname.:paramname:Thename/keyforthemetrictograbthevaluefor."""raiseNotImplementedError('get_metric')@abc.abstractmethoddefget_metric_names(self):"""Getavailablemetricnames.Getavailablemetricnames,whicharerepresentedbyasetofkeysthatcanbeusedtocheckconflictsandduplications:returns:setcontainingoneormorevaluesfrom:py:attr:nova.objects.fields.MonitorMetricType.ALL"""raiseNotImplementedError('get_metric_names')defadd_metrics_to_list(self,metrics_list):"""Addsmetricobjectstoasuppliedlistobject.:parammetric_list:nova.objects.MonitorMetricListthatthemonitorpluginshouldappendnova.objects.MonitorMetricobjectsto."""metric_names=self.get_metric_names()metrics=[]fornameinmetric_names:value,timestamp=self.get_metric(name)metric=objects.OSDMonitorMetric(name=name,value=value,timestamp=timestamp,source=self.source)metrics.append(metric)metrics_list.objects.extend(metrics)classOSDMonitorBase(MonitorBase_OSD):"""BaseclassforallmonitorsthatreturnCPU-relatedmetrics."""defget_metric_names(self):returnset([fields.OSDMonitorMetricType.OSD_NUM,])

4、创建monitor osd目录

[root@node_172_16_214_111~(keystone_admin)]#ll/usr/lib/python2.7/site-packages/nova/compute/monitors/osd/total12-rw-r--r--.1rootroot0Sep1707:35__init__.py-rw-r--r--.1rootroot157Sep1709:49__init__.pyc-rw-r--r--.1rootroot2648Sep1715:33virt_driver.py-rw-r--r--.1rootroot2711Sep1709:49virt_driver.pyc[root@node_172_16_214_111~(keystone_admin)]#cat/usr/lib/python2.7/site-packages/nova/compute/monitors/osd/virt_driver.py"""CPUmonitorbasedonvirtdrivertoretrieveCPUinformation"""importpyudevimportrefromoslo_configimportcfgfromoslo_logimportlogasloggingfromoslo_utilsimporttimeutilsfromnova.compute.monitorsimportbasefromnovaimportexceptionfromnova.i18nimport_LECONF=cfg.CONFCONF.import_opt('compute_driver','nova.virt.driver')LOG=logging.getLogger(__name__)classMonitor(base.OSDMonitorBase):"""CPUmonitorthatusesthevirtdriver'sget_host_cpu_stats()call."""def__init__(self,resource_tracker):super(Monitor,self).__init__(resource_tracker)self.source=CONF.compute_driverself.driver=resource_tracker.driverself._data={}self._cpu_stats={}defget_metric(self,name):self._update_data()returnself._data[name],self._data["timestamp"]def_update_data(self):#采集数据的就是这个函数#Don'tallowtocallthisfunctionsofrequently(<=1sec)now=timeutils.utcnow()ifself._data.get("timestamp")isnotNone:delta=now-self._data.get("timestamp")ifdelta.seconds<=1:returnself._data={}self._data["timestamp"]=now#Extractnode'sCPUstatistics.try:context=pyudev.Context()pattern=re.compile('^/dev/vd[a-z]')osd_devices=[]fordeviceincontext.list_devices(DEVTYPE='disk'):major=device['MAJOR']ifmajor=='8'orre.search(pattern,device.device_node):osd_devices.append(device.device_node)self._data["osd.num"]=len(osd_devices)except(NotImplementedError,TypeError,KeyError):LOG.exception(_LE("Notallpropertiesneededareimplemented""inthecomputedriver"))raiseexception.ResourceMonitorError(monitor=self.__class__.__name__)

5、

vim/usr/lib/python2.7/site-packages/nova/objects/__init__.py__import__('nova.objects.monitor_metric')__import__('nova.objects.monitor_osd_metric')#增加新object，跟数据库交互

6、创建新的object类

[root@node_172_16_214_111~(keystone_admin)]#cat/usr/lib/python2.7/site-packages/nova/objects/monitor_osd_metric.pyfromoslo_serializationimportjsonutilsfromoslo_utilsimporttimeutilsfromnova.objectsimportbasefromnova.objectsimportfieldsfromnovaimportutils#NOTE(jwcroppe):Usedtodeterminewhichfieldswhosevalueweneedtoadjust#(read:divideby100.0)beforesendinginformationtotheRPCnotifiersince#thesevalueswereexpectedtobewithintherange[0,1].FIELDS_REQUIRING_CONVERSION=[]@base.NovaObjectRegistry.registerclassOSDMonitorMetric(base.NovaObject):#Version1.0:Initialversion#Version1.1:AddedNUMAsupportVERSION='1.1'fields={'name':fields.OSDMonitorMetricTypeField(nullable=False),'value':fields.IntegerField(nullable=False),'numa_membw_values':fields.DictOfIntegersField(nullable=True),'timestamp':fields.DateTimeField(nullable=False),#Thiswillbethestevedoreextensionfullclassname#forthepluginfromwhichthemetricoriginates.'source':fields.StringField(nullable=False),}defobj_make_compatible(self,primitive,target_version):super(OSDMonitorMetric,self).obj_make_compatible(primitive,target_version)target_version=utils.convert_version_to_tuple(target_version)iftarget_version<(1,1)and'numa_nodes_values'inprimitive:delprimitive['numa_membw_values']#NOTE(jaypipes):Thismethodexiststoconverttheobjecttothe#formatexpectedbytheRPCnotifierformetricsevents.defto_dict(self):dict_to_return={'name':self.name,#NOTE(jaypipes):Thisiswhatjsonutils.dumps()doesto#datetime.datetimeobjects,whichiswhattimestampisin#thisobjectaswellastheoriginalsimpledictmetrics'timestamp':timeutils.strtime(self.timestamp),'source':self.source,}ifself.obj_attr_is_set('value'):ifself.nameinFIELDS_REQUIRING_CONVERSION:dict_to_return['value']=self.value/100.0else:dict_to_return['value']=self.valueelifself.obj_attr_is_set('numa_membw_values'):dict_to_return['numa_membw_values']=self.numa_membw_valuesreturndict_to_return@base.NovaObjectRegistry.registerclassOSDMonitorMetricList(base.ObjectListBase,base.NovaObject):#Version1.0:Initialversion#Version1.1:MonitorMetricversion1.1VERSION='1.1'fields={'objects':fields.ListOfObjectsField('OSDMonitorMetric'),}obj_relationships={'objects':[('1.0','1.0'),('1.1','1.1')],}@classmethoddeffrom_json(cls,metrics):"""ConvertsalegacyjsonobjectintoalistofMonitorMetricobjsandfinallyreturnsofMonitorMetricList:parammetrics:astringofjsonserializedobjects:returns:aMonitorMetricListObject."""metrics=jsonutils.loads(metrics)ifmetricselse[]metric_list=[OSDMonitorMetric(**metric)formetricinmetrics]returnOSDMonitorMetricList(objects=metric_list)#NOTE(jaypipes):Thismethodexiststoconverttheobjecttothe#formatexpectedbytheRPCnotifierformetricsevents.defto_list(self):return[m.to_dict()forminself.objects]

7、定义OSDMonitorMetricTypeField

vim/usr/lib/python2.7/site-packages/nova/objects/fields.pyclassOSDMonitorMetricTypeField(BaseEnumField):AUTO_TYPE=OSDMonitorMetricType()classOSDMonitorMetricType(Enum):OSD_NUM="osd.num"ALL=(OSD_NUM,)def__init__(self):super(OSDMonitorMetricType,self).__init__(valid_values=OSDMonitorMetricType.ALL)

8、修改metrics获取方式

vim/usr/lib/python2.7/site-packages/nova/scheduler/host_manager.pydefupdate_from_compute_node(self,compute):#self.metrics=objects.MonitorMetricList.from_json(compute.metrics)#先注释掉cpumonitorself.metrics=objects.OSDMonitorMetricList.from_json(compute.metrics)vim/usr/lib/python2.7/site-packages/nova/compute/resource_tracker.py#先注释掉cpumonitordef_get_host_metrics(self,context,nodename):"""Getthemetricsfrommonitorsandnotifyinformationtomessagebus."""#metrics=objects.MonitorMetricList()metrics=objects.OSDMonitorMetricList()

9、修改nova配置文件

vim/etc/nova/nova.conf[METRICS]#determinehowmetricsareweighed:score=-1.0*cpu.percent##thelistformatisweight_setting=name1=1.0,name2=-1.0weight_setting=osd.num=5.0[DEFAULT]compute_monitors=osd.virt_driverscheduler_weight_classes=nova.scheduler.weights.metrics.MetricsWeigherscheduler_host_subset_size=1

scheduler知识点小记：

/usr/lib/python2.7/site-packages/nova/scheduler/driver.py#默认host_managerdriver是nova.scheduler.host_manager.HostManagerscheduler_driver_opts=[cfg.StrOpt('scheduler_host_manager',default='nova.scheduler.host_manager.HostManager',help='Theschedulerhostmanagerclasstouse'),]/usr/lib/python2.7/site-packages/nova/scheduler/manager.py#默认schedulerdriver是nova.scheduler.filter_scheduler.FilterSchedulerscheduler_driver_opts=[cfg.StrOpt('scheduler_driver',default='nova.scheduler.filter_scheduler.FilterScheduler',help='Defaultdrivertouseforthescheduler'),cfg.IntOpt('scheduler_driver_task_period',default=60,help='Howoften(inseconds)torunperiodictasksin''theschedulerdriverofyourchoice.''Pleasenotethisislikelytointeractwiththevalue''ofservice_down_time,butexactlyhowtheyinteract''willdependonyourchoiceofschedulerdriver.'),]/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py#过滤出主机节点def_schedule(self,context,request_spec,filter_properties):"""Returnsalistofhoststhatmeettherequiredspecs,orderedbytheirfitness."""elevated=context.elevated()instance_properties=request_spec['instance_properties']#NOTE(danms):Instancehereisstilladict,whichisconvertedfrom#anobject.Thepci_requestsareadictaswell.Convertthiswhen#wegetanobjectallthewaytothispath.#TODO(sbauza):WillbefixedlaterbytheRequestSpecobjectpci_requests=instance_properties.get('pci_requests')ifpci_requests:pci_requests=(objects.InstancePCIRequests.from_request_spec_instance_props(pci_requests))instance_properties['pci_requests']=pci_requestsinstance_type=request_spec.get("instance_type",None)update_group_hosts=filter_properties.get('group_updated',False)config_options=self._get_configuration_options()filter_properties.update({'context':context,'request_spec':request_spec,'config_options':config_options,'instance_type':instance_type})#Findourlocallistofacceptablehostsbyrepeatedly#filteringandweighingouroptions.Eachtimewechoosea#host,wevirtuallyconsumeresourcesonitsosubsequent#selectionscanadjustaccordingly.#Note:remember,weareusinganiteratorhere.Soonly#traversethislistonce.Thiscanbiteyouifthehosts#arebeingscannedinafilterorweighingfunction.hosts=self._get_all_host_states(elevated)selected_hosts=[]num_instances=request_spec.get('num_instances',1)fornuminrange(num_instances):#Filterlocalhostsbasedonrequirements...hosts=self.host_manager.get_filtered_hosts(hosts,#先经过filterdriver过滤filter_properties,index=num)ifnothosts:#Can'tgetanymorelocally.breakLOG.debug("Filtered%(hosts)s",{'hosts':hosts})weighed_hosts=self.host_manager.get_weighed_hosts(hosts,#然后再计算hosts权重列表(从大到小排列)filter_properties)LOG.debug("Weighed%(hosts)s",{'hosts':weighed_hosts})scheduler_host_subset_size=CONF.scheduler_host_subset_sizeifscheduler_host_subset_size>len(weighed_hosts):scheduler_host_subset_size=len(weighed_hosts)ifscheduler_host_subset_size<1:scheduler_host_subset_size=1chosen_host=random.choice(weighed_hosts[0:scheduler_host_subset_size])LOG.debug("Selectedhost:%(host)s",{'host':chosen_host})selected_hosts.append(chosen_host)#Nowconsumetheresourcessothefilter/weights#willchangeforthenextinstance.chosen_host.obj.consume_from_instance(instance_properties)ifupdate_group_hostsisTrue:#NOTE(sbauza):Groupdetailsareserializedintoalistnow#thattheyarepopulatedbytheconductor,weneedto#deserializethemifisinstance(filter_properties['group_hosts'],list):filter_properties['group_hosts']=set(filter_properties['group_hosts'])filter_properties['group_hosts'].add(chosen_host.obj.host)returnselected_hosts

参考链接

https://01.org/sites/default/files/utilization_based_scheduing_in_openstack_compute_nova-revision002.pdf