Message-Id: <20170922035850.6689-1-kys@exchange.microsoft.com>
Date:   Thu, 21 Sep 2017 20:58:49 -0700
From:   kys@...hange.microsoft.com
To:     gregkh@...uxfoundation.org, linux-kernel@...r.kernel.org,
        devel@...uxdriverproject.org, olaf@...fle.de, apw@...onical.com,
        vkuznets@...hat.com, jasowang@...hat.com,
        leann.ogasawara@...onical.com, marcelo.cerri@...onical.com,
        sthemmin@...rosoft.com
Cc:     Stephen Hemminger <stephen@...workplumber.org>,
        "K . Y . Srinivasan" <kys@...rosoft.com>
Subject: [PATCH 1/2] vmbus: add per-channel sysfs info

From: Stephen Hemminger <stephen@...workplumber.org>

This extends existing vmbus related sysfs structure to provide per-channel
state information. This is useful when diagnosing issues with multiple
queues in networking and storage.

The existing sysfs only displayed information about the primary
channel. The one place it reported multiple channels was the
channel_vp_mapping file which violated the sysfs convention
of one value per file.

Signed-off-by: Stephen Hemminger <sthemmin@...rosoft.com>
Signed-off-by: K. Y. Srinivasan <kys@...rosoft.com>
---
 Documentation/ABI/stable/sysfs-bus-vmbus |  56 ++++++++++
 drivers/hv/channel_mgmt.c                |  10 +-
 drivers/hv/hyperv_vmbus.h                |   2 +
 drivers/hv/vmbus_drv.c                   | 185 +++++++++++++++++++++++++++++--
 include/linux/hyperv.h                   |   6 +
 5 files changed, 246 insertions(+), 13 deletions(-)

diff --git a/Documentation/ABI/stable/sysfs-bus-vmbus b/Documentation/ABI/stable/sysfs-bus-vmbus
index 5d0125f7bcaf..0ebd8a1537a0 100644
--- a/Documentation/ABI/stable/sysfs-bus-vmbus
+++ b/Documentation/ABI/stable/sysfs-bus-vmbus
@@ -41,3 +41,59 @@ KernelVersion:	4.5
 Contact:	K. Y. Srinivasan <kys@...rosoft.com>
 Description:	The 16 bit vendor ID of the device
 Users:		tools/hv/lsvmbus and user level RDMA libraries
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/cpu
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	VCPU (sub)channel is affinitized to
+Users:		tools/hv/lsvmbus and other debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/cpu
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	VCPU (sub)channel is affinitized to
+Users:		tools/hv/lsvmbus and other debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/in_mask
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Inbound channel signaling state
+Users:		Debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/latency
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Channel signaling latency
+Users:		Debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/out_mask
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Outbound channel signaling state
+Users:		Debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/pending
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Channel interrupt pending state
+Users:		Debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/read_avail
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Bytes availabble to read
+Users:		Debuggig tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/write_avail
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@...rosoft.com>
+Description:	Bytes availabble to write
+Users:		Debuggig tools
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index 624d815745e4..747887038992 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -350,7 +350,7 @@ static void free_channel(struct vmbus_channel *channel)
 {
 	tasklet_kill(&channel->callback_event);
 
-	kfree_rcu(channel, rcu);
+	kobject_put(&channel->kobj);
 }
 
 static void percpu_channel_enq(void *arg)
@@ -513,6 +513,14 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 	newchannel->state = CHANNEL_OPEN_STATE;
 
 	if (!fnew) {
+		struct hv_device *dev
+			= newchannel->primary_channel->device_obj;
+
+		if (vmbus_add_channel_kobj(dev, newchannel)) {
+			atomic_dec(&vmbus_connection.offer_in_progress);
+			goto err_free_chan;
+		}
+
 		if (channel->sc_creation_callback != NULL)
 			channel->sc_creation_callback(newchannel);
 		return;
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 1b6a5e0dfa75..bfa2d0af4e1d 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -384,6 +384,8 @@ struct hv_device *vmbus_device_create(const uuid_le *type,
 
 int vmbus_device_register(struct hv_device *child_device_obj);
 void vmbus_device_unregister(struct hv_device *device_obj);
+int vmbus_add_channel_kobj(struct hv_device *device_obj,
+			   struct vmbus_channel *channel);
 
 struct vmbus_channel *relid2channel(u32 relid);
 
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 43160a2eafe0..c126d3937414 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -107,28 +107,30 @@ static void print_alias_name(struct hv_device *hv_dev, char *alias_name)
 		sprintf(&alias_name[i], "%02x", hv_dev->dev_type.b[i/2]);
 }
 
-static u8 channel_monitor_group(struct vmbus_channel *channel)
+static u8 channel_monitor_group(const struct vmbus_channel *channel)
 {
 	return (u8)channel->offermsg.monitorid / 32;
 }
 
-static u8 channel_monitor_offset(struct vmbus_channel *channel)
+static u8 channel_monitor_offset(const struct vmbus_channel *channel)
 {
 	return (u8)channel->offermsg.monitorid % 32;
 }
 
-static u32 channel_pending(struct vmbus_channel *channel,
-			   struct hv_monitor_page *monitor_page)
+static u32 channel_pending(const struct vmbus_channel *channel,
+			   const struct hv_monitor_page *monitor_page)
 {
 	u8 monitor_group = channel_monitor_group(channel);
+
 	return monitor_page->trigger_group[monitor_group].pending;
 }
 
-static u32 channel_latency(struct vmbus_channel *channel,
-			   struct hv_monitor_page *monitor_page)
+static u32 channel_latency(const struct vmbus_channel *channel,
+			   const struct hv_monitor_page *monitor_page)
 {
 	u8 monitor_group = channel_monitor_group(channel);
 	u8 monitor_offset = channel_monitor_offset(channel);
+
 	return monitor_page->latency[monitor_group][monitor_offset];
 }
 
@@ -1134,6 +1136,145 @@ void vmbus_driver_unregister(struct hv_driver *hv_driver)
 }
 EXPORT_SYMBOL_GPL(vmbus_driver_unregister);
 
+
+/*
+ * Called when last reference to channel is gone.
+ */
+static void vmbus_chan_release(struct kobject *kobj)
+{
+	struct vmbus_channel *channel
+		= container_of(kobj, struct vmbus_channel, kobj);
+
+	kfree_rcu(channel, rcu);
+}
+
+struct vmbus_chan_attribute {
+	struct attribute attr;
+	ssize_t (*show)(const struct vmbus_channel *chan, char *buf);
+	ssize_t (*store)(struct vmbus_channel *chan,
+			 const char *buf, size_t count);
+};
+#define VMBUS_CHAN_ATTR(_name, _mode, _show, _store) \
+	struct vmbus_chan_attribute chan_attr_##_name \
+		= __ATTR(_name, _mode, _show, _store)
+#define VMBUS_CHAN_ATTR_RW(_name) \
+	struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RW(_name)
+#define VMBUS_CHAN_ATTR_RO(_name) \
+	struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RO(_name)
+#define VMBUS_CHAN_ATTR_WO(_name) \
+	struct vmbus_chan_attribute chan_attr_##_name = __ATTR_WO(_name)
+
+static ssize_t vmbus_chan_attr_show(struct kobject *kobj,
+				    struct attribute *attr, char *buf)
+{
+	const struct vmbus_chan_attribute *attribute
+		= container_of(attr, struct vmbus_chan_attribute, attr);
+	const struct vmbus_channel *chan
+		= container_of(kobj, struct vmbus_channel, kobj);
+
+	if (!attribute->show)
+		return -EIO;
+
+	return attribute->show(chan, buf);
+}
+
+static const struct sysfs_ops vmbus_chan_sysfs_ops = {
+	.show = vmbus_chan_attr_show,
+};
+
+static ssize_t out_mask_show(const struct vmbus_channel *channel, char *buf)
+{
+	const struct hv_ring_buffer_info *rbi = &channel->outbound;
+
+	return sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
+}
+VMBUS_CHAN_ATTR_RO(out_mask);
+
+static ssize_t in_mask_show(const struct vmbus_channel *channel, char *buf)
+{
+	const struct hv_ring_buffer_info *rbi = &channel->inbound;
+
+	return sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
+}
+VMBUS_CHAN_ATTR_RO(in_mask);
+
+static ssize_t read_avail_show(const struct vmbus_channel *channel, char *buf)
+{
+	const struct hv_ring_buffer_info *rbi = &channel->inbound;
+
+	return sprintf(buf, "%u\n", hv_get_bytes_to_read(rbi));
+}
+VMBUS_CHAN_ATTR_RO(read_avail);
+
+static ssize_t write_avail_show(const struct vmbus_channel *channel, char *buf)
+{
+	const struct hv_ring_buffer_info *rbi = &channel->outbound;
+
+	return sprintf(buf, "%u\n", hv_get_bytes_to_write(rbi));
+}
+VMBUS_CHAN_ATTR_RO(write_avail);
+
+static ssize_t show_target_cpu(const struct vmbus_channel *channel, char *buf)
+{
+	return sprintf(buf, "%u\n", channel->target_cpu);
+}
+VMBUS_CHAN_ATTR(cpu, S_IRUGO, show_target_cpu, NULL);
+
+static ssize_t channel_pending_show(const struct vmbus_channel *channel,
+				    char *buf)
+{
+	return sprintf(buf, "%d\n",
+		       channel_pending(channel,
+				       vmbus_connection.monitor_pages[1]));
+}
+VMBUS_CHAN_ATTR(pending, S_IRUGO, channel_pending_show, NULL);
+
+static ssize_t channel_latency_show(const struct vmbus_channel *channel,
+				    char *buf)
+{
+	return sprintf(buf, "%d\n",
+		       channel_latency(channel,
+				       vmbus_connection.monitor_pages[1]));
+}
+VMBUS_CHAN_ATTR(latency, S_IRUGO, channel_latency_show, NULL);
+
+static struct attribute *vmbus_chan_attrs[] = {
+	&chan_attr_out_mask.attr,
+	&chan_attr_in_mask.attr,
+	&chan_attr_read_avail.attr,
+	&chan_attr_write_avail.attr,
+	&chan_attr_cpu.attr,
+	&chan_attr_pending.attr,
+	&chan_attr_latency.attr,
+	NULL
+};
+
+static struct kobj_type vmbus_chan_ktype = {
+	.sysfs_ops = &vmbus_chan_sysfs_ops,
+	.release = vmbus_chan_release,
+	.default_attrs = vmbus_chan_attrs,
+};
+
+/*
+ * vmbus_add_channel_kobj - setup a sub-directory under device/channels
+ */
+int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel)
+{
+	struct kobject *kobj = &channel->kobj;
+	u32 relid = channel->offermsg.child_relid;
+	int ret;
+
+	kobj->kset = dev->channels_kset;
+	ret = kobject_init_and_add(kobj, &vmbus_chan_ktype, NULL,
+				   "%u", relid);
+	if (ret)
+		return ret;
+
+	kobject_uevent(kobj, KOBJ_ADD);
+
+	return 0;
+}
+
 /*
  * vmbus_device_create - Creates and registers a new child device
  * on the vmbus.
@@ -1165,7 +1306,8 @@ struct hv_device *vmbus_device_create(const uuid_le *type,
  */
 int vmbus_device_register(struct hv_device *child_device_obj)
 {
-	int ret = 0;
+	struct kobject *kobj = &child_device_obj->device.kobj;
+	int ret;
 
 	dev_set_name(&child_device_obj->device, "%pUl",
 		     child_device_obj->channel->offermsg.offer.if_instance.b);
@@ -1179,13 +1321,32 @@ int vmbus_device_register(struct hv_device *child_device_obj)
 	 * binding...which will eventually call vmbus_match() and vmbus_probe()
 	 */
 	ret = device_register(&child_device_obj->device);
-
-	if (ret)
+	if (ret) {
 		pr_err("Unable to register child device\n");
-	else
-		pr_debug("child device %s registered\n",
-			dev_name(&child_device_obj->device));
+		return ret;
+	}
+
+	child_device_obj->channels_kset = kset_create_and_add("channels",
+							      NULL, kobj);
+	if (!child_device_obj->channels_kset) {
+		ret = -ENOMEM;
+		goto err_dev_unregister;
+	}
+
+	ret = vmbus_add_channel_kobj(child_device_obj,
+				     child_device_obj->channel);
+	if (ret) {
+		pr_err("Unable to register primary channeln");
+		goto err_kset_unregister;
+	}
+
+	return 0;
+
+err_kset_unregister:
+	kset_unregister(child_device_obj->channels_kset);
 
+err_dev_unregister:
+	device_unregister(&child_device_obj->device);
 	return ret;
 }
 
diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h
index 07650d0232cc..91c88e0e771e 100644
--- a/include/linux/hyperv.h
+++ b/include/linux/hyperv.h
@@ -841,6 +841,11 @@ struct vmbus_channel {
 	 */
 	struct rcu_head rcu;
 
+	/*
+	 * For sysfs per-channel properties.
+	 */
+	struct kobject			kobj;
+
 	/*
 	 * For performance critical channels (storage, networking
 	 * etc,), Hyper-V has a mechanism to enhance the throughput
@@ -1123,6 +1128,7 @@ struct hv_device {
 	struct device device;
 
 	struct vmbus_channel *channel;
+	struct kset	     *channels_kset;
 };
 
 
-- 
2.14.1

Powered by blists - more mailing lists