Date:   Mon, 23 Jan 2023 16:00:10 -0600
From:   Steve Wahl <steve.wahl@....com>
To:     Steve Wahl <steve.wahl@....com>,
        Dimitri Sivanich <dimitri.sivanich@....com>,
        Russ Anderson <russ.anderson@....com>,
        Thomas Gleixner <tglx@...utronix.de>,
        Ingo Molnar <mingo@...hat.com>, Borislav Petkov <bp@...en8.de>,
        Dave Hansen <dave.hansen@...ux.intel.com>, x86@...nel.org,
        "H. Peter Anvin" <hpa@...or.com>, linux-kernel@...r.kernel.org
Subject: Re: [PATCH] UV support for sub-NUMA clustering

PLEASE IGNORE.  I forgot to put x86/<etc> in the subject line.  Let me
fix it!!

Thanks.

--> Steve Wahl


On Mon, Jan 23, 2023 at 03:57:21PM -0600, Steve Wahl wrote:
> Sub-NUMA clustering (SNC) invalidates previous assumptions of a 1:1
> relationship between blades, sockets, and nodes.  Fix these
> assumptions and build tables correctly when SNC is enabled.
> 
> Also replace uses of BUG() and BUG_ON() with WARN_ON() and recovery.
> 
> Signed-off-by: Steve Wahl <steve.wahl@....com>
> ---
>  arch/x86/include/asm/uv/uv_hub.h   |  32 ++--
>  arch/x86/kernel/apic/x2apic_uv_x.c | 245 ++++++++++++++++-------------
>  2 files changed, 160 insertions(+), 117 deletions(-)
> 
> diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
> index d3e3197917be..5fa76c2ced51 100644
> --- a/arch/x86/include/asm/uv/uv_hub.h
> +++ b/arch/x86/include/asm/uv/uv_hub.h
> @@ -177,6 +177,7 @@ struct uv_hub_info_s {
>  	unsigned short		nr_possible_cpus;
>  	unsigned short		nr_online_cpus;
>  	short			memory_nid;
> +	unsigned short		*node_to_socket;
>  };
>  
>  /* CPU specific info with a pointer to the hub common info struct */
> @@ -519,25 +520,30 @@ static inline int uv_socket_to_node(int socket)
>  	return _uv_socket_to_node(socket, uv_hub_info->socket_to_node);
>  }
>  
> +static inline int uv_pnode_to_socket(int pnode)
> +{
> +	unsigned short *p2s = uv_hub_info->pnode_to_socket;
> +
> +	return p2s ? p2s[pnode - uv_hub_info->min_pnode] : pnode;
> +}
> +
>  /* pnode, offset --> socket virtual */
>  static inline void *uv_pnode_offset_to_vaddr(int pnode, unsigned long offset)
>  {
>  	unsigned int m_val = uv_hub_info->m_val;
>  	unsigned long base;
> -	unsigned short sockid, node, *p2s;
> +	unsigned short sockid;
>  
>  	if (m_val)
>  		return __va(((unsigned long)pnode << m_val) | offset);
>  
> -	p2s = uv_hub_info->pnode_to_socket;
> -	sockid = p2s ? p2s[pnode - uv_hub_info->min_pnode] : pnode;
> -	node = uv_socket_to_node(sockid);
> +	sockid = uv_pnode_to_socket(pnode);
>  
>  	/* limit address of previous socket is our base, except node 0 is 0 */
> -	if (!node)
> +	if (sockid == 0)
>  		return __va((unsigned long)offset);
>  
> -	base = (unsigned long)(uv_hub_info->gr_table[node - 1].limit);
> +	base = (unsigned long)(uv_hub_info->gr_table[sockid - 1].limit);
>  	return __va(base << UV_GAM_RANGE_SHFT | offset);
>  }
>  
> @@ -644,7 +650,7 @@ static inline int uv_cpu_blade_processor_id(int cpu)
>  /* Blade number to Node number (UV2..UV4 is 1:1) */
>  static inline int uv_blade_to_node(int blade)
>  {
> -	return blade;
> +	return uv_socket_to_node(blade);
>  }
>  
>  /* Blade number of current cpu. Numnbered 0 .. <#blades -1> */
> @@ -656,23 +662,27 @@ static inline int uv_numa_blade_id(void)
>  /*
>   * Convert linux node number to the UV blade number.
>   * .. Currently for UV2 thru UV4 the node and the blade are identical.
> - * .. If this changes then you MUST check references to this function!
> + * .. UV5 needs conversion when sub-numa clustering is enabled.
>   */
>  static inline int uv_node_to_blade_id(int nid)
>  {
> -	return nid;
> +	unsigned short *n2s = uv_hub_info->node_to_socket;
> +
> +	return n2s ? n2s[nid] : nid;
>  }
>  
>  /* Convert a CPU number to the UV blade number */
>  static inline int uv_cpu_to_blade_id(int cpu)
>  {
> -	return uv_node_to_blade_id(cpu_to_node(cpu));
> +	return uv_cpu_hub_info(cpu)->numa_blade_id;
>  }
>  
>  /* Convert a blade id to the PNODE of the blade */
>  static inline int uv_blade_to_pnode(int bid)
>  {
> -	return uv_hub_info_list(uv_blade_to_node(bid))->pnode;
> +	unsigned short *s2p = uv_hub_info->socket_to_pnode;
> +
> +	return s2p ? s2p[bid] : bid;
>  }
>  
>  /* Nid of memory node on blade. -1 if no blade-local memory */
> diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
> index 482855227964..57bfc40470eb 100644
> --- a/arch/x86/kernel/apic/x2apic_uv_x.c
> +++ b/arch/x86/kernel/apic/x2apic_uv_x.c
> @@ -546,7 +546,6 @@ unsigned long sn_rtc_cycles_per_second;
>  EXPORT_SYMBOL(sn_rtc_cycles_per_second);
>  
>  /* The following values are used for the per node hub info struct */
> -static __initdata unsigned short		*_node_to_pnode;
>  static __initdata unsigned short		_min_socket, _max_socket;
>  static __initdata unsigned short		_min_pnode, _max_pnode, _gr_table_len;
>  static __initdata struct uv_gam_range_entry	*uv_gre_table;
> @@ -554,6 +553,7 @@ static __initdata struct uv_gam_parameters	*uv_gp_table;
>  static __initdata unsigned short		*_socket_to_node;
>  static __initdata unsigned short		*_socket_to_pnode;
>  static __initdata unsigned short		*_pnode_to_socket;
> +static __initdata unsigned short		*_node_to_socket;
>  
>  static __initdata struct uv_gam_range_s		*_gr_table;
>  
> @@ -617,7 +617,9 @@ static __init void build_uv_gr_table(void)
>  
>  	bytes = _gr_table_len * sizeof(struct uv_gam_range_s);
>  	grt = kzalloc(bytes, GFP_KERNEL);
> -	BUG_ON(!grt);
> +	WARN_ON_ONCE(!grt);
> +	if (!grt)
> +		return;
>  	_gr_table = grt;
>  
>  	for (; gre->type != UV_GAM_RANGE_TYPE_UNUSED; gre++) {
> @@ -1292,6 +1294,7 @@ static void __init uv_init_hub_info(struct uv_hub_info_s *hi)
>  	hi->nasid_shift		= uv_cpuid.nasid_shift;
>  	hi->min_pnode		= _min_pnode;
>  	hi->min_socket		= _min_socket;
> +	hi->node_to_socket	= _node_to_socket;
>  	hi->pnode_to_socket	= _pnode_to_socket;
>  	hi->socket_to_node	= _socket_to_node;
>  	hi->socket_to_pnode	= _socket_to_pnode;
> @@ -1490,16 +1493,56 @@ static __init void boot_init_possible_blades(struct uv_hub_info_s *hub_info)
>  	pr_info("UV: number nodes/possible blades %d\n", uv_pb);
>  }
>  
> +static int __init alloc_conv_table(int num_elem, unsigned short **table)
> +{
> +	int i;
> +	size_t bytes;
> +
> +	bytes = num_elem * sizeof(*table[0]);
> +	*table = kmalloc(bytes, GFP_KERNEL);
> +	WARN_ON_ONCE(!*table);
> +	if (!*table)
> +		return -ENOMEM;
> +	for (i = 0; i < num_elem; i++)
> +		((unsigned short *)*table)[i] = SOCK_EMPTY;
> +	return 0;
> +}
> +
> +/* Remove conversion table if it's 1:1 */
> +#define FREE_1_TO_1_TABLE(tbl, min, max, max2) free_1_to_1_table(&tbl, #tbl, min, max, max2)
> +
> +static void __init free_1_to_1_table(unsigned short **tp, char *tname, int min, int max, int max2)
> +{
> +	int i;
> +	unsigned short *table = *tp;
> +
> +	if (table == NULL)
> +		return;
> +	if (max != max2)
> +		return;
> +	for (i = 0; i < max; i++) {
> +		if (i != table[i])
> +			return;
> +	}
> +	kfree(table);
> +	*tp = NULL;
> +	pr_info("UV: %s is 1:1, conversion table removed\n", tname);
> +}
> +
> +/*
> + * Build Socket Tables
> + * If the number of nodes is >1 per socket, socket to node table will
> + * contain lowest node number on that socket.
> + */
>  static void __init build_socket_tables(void)
>  {
>  	struct uv_gam_range_entry *gre = uv_gre_table;
> -	int num, nump;
> +	int nums, numn, nump;
>  	int cpu, i, lnid;
>  	int minsock = _min_socket;
>  	int maxsock = _max_socket;
>  	int minpnode = _min_pnode;
>  	int maxpnode = _max_pnode;
> -	size_t bytes;
>  
>  	if (!gre) {
>  		if (is_uv2_hub() || is_uv3_hub()) {
> @@ -1507,39 +1550,36 @@ static void __init build_socket_tables(void)
>  			return;
>  		}
>  		pr_err("UV: Error: UVsystab address translations not available!\n");
> -		BUG();
> +		WARN_ON_ONCE(!gre);
> +		return;
>  	}
>  
> -	/* Build socket id -> node id, pnode */
> -	num = maxsock - minsock + 1;
> -	bytes = num * sizeof(_socket_to_node[0]);
> -	_socket_to_node = kmalloc(bytes, GFP_KERNEL);
> -	_socket_to_pnode = kmalloc(bytes, GFP_KERNEL);
> -
> +	numn = num_possible_nodes();
>  	nump = maxpnode - minpnode + 1;
> -	bytes = nump * sizeof(_pnode_to_socket[0]);
> -	_pnode_to_socket = kmalloc(bytes, GFP_KERNEL);
> -	BUG_ON(!_socket_to_node || !_socket_to_pnode || !_pnode_to_socket);
> +	nums = maxsock - minsock + 1;
>  
> -	for (i = 0; i < num; i++)
> -		_socket_to_node[i] = _socket_to_pnode[i] = SOCK_EMPTY;
> +	/* Allocate and clear tables */
> +	if (alloc_conv_table(nump, &_pnode_to_socket) < 0)
> +		return;
> +	if (alloc_conv_table(nums, &_socket_to_pnode) < 0)
> +		return;
>  
> -	for (i = 0; i < nump; i++)
> -		_pnode_to_socket[i] = SOCK_EMPTY;
> +	if (alloc_conv_table(numn, &_node_to_socket) < 0)
> +		return;
> +	if (alloc_conv_table(nums, &_socket_to_node) < 0)
> +		return;
>  
>  	/* Fill in pnode/node/addr conversion list values: */
> -	pr_info("UV: GAM Building socket/pnode conversion tables\n");
>  	for (; gre->type != UV_GAM_RANGE_TYPE_UNUSED; gre++) {
>  		if (gre->type == UV_GAM_RANGE_TYPE_HOLE)
>  			continue;
>  		i = gre->sockid - minsock;
> -		/* Duplicate: */
> -		if (_socket_to_pnode[i] != SOCK_EMPTY)
> -			continue;
> -		_socket_to_pnode[i] = gre->pnode;
> +		if (_socket_to_pnode[i] == SOCK_EMPTY)
> +			_socket_to_pnode[i] = gre->pnode;
>  
>  		i = gre->pnode - minpnode;
> -		_pnode_to_socket[i] = gre->sockid;
> +		if (_pnode_to_socket[i] == SOCK_EMPTY)
> +			_pnode_to_socket[i] = gre->sockid;
>  
>  		pr_info("UV: sid:%02x type:%d nasid:%04x pn:%02x pn2s:%2x\n",
>  			gre->sockid, gre->type, gre->nasid,
> @@ -1549,66 +1589,39 @@ static void __init build_socket_tables(void)
>  
>  	/* Set socket -> node values: */
>  	lnid = NUMA_NO_NODE;
> -	for_each_present_cpu(cpu) {
> +	for_each_possible_cpu(cpu) {
>  		int nid = cpu_to_node(cpu);
>  		int apicid, sockid;
>  
>  		if (lnid == nid)
>  			continue;
>  		lnid = nid;
> +
>  		apicid = per_cpu(x86_cpu_to_apicid, cpu);
>  		sockid = apicid >> uv_cpuid.socketid_shift;
> -		_socket_to_node[sockid - minsock] = nid;
> -		pr_info("UV: sid:%02x: apicid:%04x node:%2d\n",
> -			sockid, apicid, nid);
> -	}
>  
> -	/* Set up physical blade to pnode translation from GAM Range Table: */
> -	bytes = num_possible_nodes() * sizeof(_node_to_pnode[0]);
> -	_node_to_pnode = kmalloc(bytes, GFP_KERNEL);
> -	BUG_ON(!_node_to_pnode);
> +		if (_socket_to_node[sockid - minsock] == SOCK_EMPTY)
> +			_socket_to_node[sockid - minsock] = nid;
>  
> -	for (lnid = 0; lnid < num_possible_nodes(); lnid++) {
> -		unsigned short sockid;
> +		if (_node_to_socket[nid] == SOCK_EMPTY)
> +			_node_to_socket[nid] = sockid;
>  
> -		for (sockid = minsock; sockid <= maxsock; sockid++) {
> -			if (lnid == _socket_to_node[sockid - minsock]) {
> -				_node_to_pnode[lnid] = _socket_to_pnode[sockid - minsock];
> -				break;
> -			}
> -		}
> -		if (sockid > maxsock) {
> -			pr_err("UV: socket for node %d not found!\n", lnid);
> -			BUG();
> -		}
> +		pr_info("UV: sid:%02x: apicid:%04x socket:%02d node:%03x s2n:%03x\n",
> +			sockid,
> +			apicid,
> +			_node_to_socket[nid],
> +			nid,
> +			_socket_to_node[sockid - minsock]);
>  	}
>  
>  	/*
> -	 * If socket id == pnode or socket id == node for all nodes,
> +	 * If e.g. socket id == pnode for all pnodes,
>  	 *   system runs faster by removing corresponding conversion table.
>  	 */
> -	pr_info("UV: Checking socket->node/pnode for identity maps\n");
> -	if (minsock == 0) {
> -		for (i = 0; i < num; i++)
> -			if (_socket_to_node[i] == SOCK_EMPTY || i != _socket_to_node[i])
> -				break;
> -		if (i >= num) {
> -			kfree(_socket_to_node);
> -			_socket_to_node = NULL;
> -			pr_info("UV: 1:1 socket_to_node table removed\n");
> -		}
> -	}
> -	if (minsock == minpnode) {
> -		for (i = 0; i < num; i++)
> -			if (_socket_to_pnode[i] != SOCK_EMPTY &&
> -				_socket_to_pnode[i] != i + minpnode)
> -				break;
> -		if (i >= num) {
> -			kfree(_socket_to_pnode);
> -			_socket_to_pnode = NULL;
> -			pr_info("UV: 1:1 socket_to_pnode table removed\n");
> -		}
> -	}
> +	FREE_1_TO_1_TABLE(_socket_to_node, _min_socket, nums, numn);
> +	FREE_1_TO_1_TABLE(_node_to_socket, _min_socket, nums, numn);
> +	FREE_1_TO_1_TABLE(_socket_to_pnode, _min_pnode, nums, nump);
> +	FREE_1_TO_1_TABLE(_pnode_to_socket, _min_pnode, nums, nump);
>  }
>  
>  /* Check which reboot to use */
> @@ -1692,12 +1705,13 @@ static __init int uv_system_init_hubless(void)
>  static void __init uv_system_init_hub(void)
>  {
>  	struct uv_hub_info_s hub_info = {0};
> -	int bytes, cpu, nodeid;
> +	int bytes, cpu, nodeid, bid;
>  	unsigned short min_pnode = 9999, max_pnode = 0;
>  	char *hub = is_uv5_hub() ? "UV500" :
>  		    is_uv4_hub() ? "UV400" :
>  		    is_uv3_hub() ? "UV300" :
>  		    is_uv2_hub() ? "UV2000/3000" : NULL;
> +	struct uv_hub_info_s **uv_hub_info_list_blade;
>  
>  	if (!hub) {
>  		pr_err("UV: Unknown/unsupported UV hub\n");
> @@ -1720,9 +1734,12 @@ static void __init uv_system_init_hub(void)
>  	build_uv_gr_table();
>  	set_block_size();
>  	uv_init_hub_info(&hub_info);
> -	uv_possible_blades = num_possible_nodes();
> -	if (!_node_to_pnode)
> +	/* If UV2 or UV3 may need to get # blades from HW */
> +	if (is_uv(UV2|UV3) && !uv_gre_table)
>  		boot_init_possible_blades(&hub_info);
> +	else
> +		/* min/max sockets set in decode_gam_rng_tbl */
> +		uv_possible_blades = (_max_socket - _min_socket) + 1;
>  
>  	/* uv_num_possible_blades() is really the hub count: */
>  	pr_info("UV: Found %d hubs, %d nodes, %d CPUs\n", uv_num_possible_blades(), num_possible_nodes(), num_possible_cpus());
> @@ -1731,79 +1748,92 @@ static void __init uv_system_init_hub(void)
>  	hub_info.coherency_domain_number = sn_coherency_id;
>  	uv_rtc_init();
>  
> +	/*
> +	 * __uv_hub_info_list[] is indexed by node, but there is only one hub_info
> +	 * structure per blade.  First, allocate one structure per blade.
> +	 */
> +
>  	bytes = sizeof(void *) * uv_num_possible_blades();
> -	__uv_hub_info_list = kzalloc(bytes, GFP_KERNEL);
> -	BUG_ON(!__uv_hub_info_list);
> +	uv_hub_info_list_blade = kzalloc(bytes, GFP_KERNEL);
> +	WARN_ON_ONCE(!uv_hub_info_list_blade);
> +	if (!uv_hub_info_list_blade)
> +		return;
>  
>  	bytes = sizeof(struct uv_hub_info_s);
> -	for_each_node(nodeid) {
> +	for_each_possible_blade(bid) {
>  		struct uv_hub_info_s *new_hub;
>  
> -		if (__uv_hub_info_list[nodeid]) {
> -			pr_err("UV: Node %d UV HUB already initialized!?\n", nodeid);
> -			BUG();
> -		}
> -
> -		/* Allocate new per hub info list */
> -		new_hub = (nodeid == 0) ?  &uv_hub_info_node0 : kzalloc_node(bytes, GFP_KERNEL, nodeid);
> -		BUG_ON(!new_hub);
> -		__uv_hub_info_list[nodeid] = new_hub;
> -		new_hub = uv_hub_info_list(nodeid);
> -		BUG_ON(!new_hub);
> +		/* Allocate & fill new per hub info list */
> +		new_hub = (bid == 0) ?  &uv_hub_info_node0
> +			: kzalloc_node(bytes, GFP_KERNEL, uv_blade_to_node(bid));
> +		WARN_ON_ONCE(!new_hub);
> +		if (!new_hub)
> +			return;
> +		uv_hub_info_list_blade[bid] = new_hub;
>  		*new_hub = hub_info;
>  
>  		/* Use information from GAM table if available: */
> -		if (_node_to_pnode)
> -			new_hub->pnode = _node_to_pnode[nodeid];
> +		if (uv_gre_table)
> +			new_hub->pnode = uv_blade_to_pnode(bid);
>  		else /* Or fill in during CPU loop: */
>  			new_hub->pnode = 0xffff;
>  
> -		new_hub->numa_blade_id = uv_node_to_blade_id(nodeid);
> +		new_hub->numa_blade_id = bid;
>  		new_hub->memory_nid = NUMA_NO_NODE;
>  		new_hub->nr_possible_cpus = 0;
>  		new_hub->nr_online_cpus = 0;
>  	}
>  
> +	/*
> +	 * Now populate __uv_hub_info_list[] for each node with the
> +	 * pointer to the struct for the blade it resides on.
> +	 */
> +
> +	bytes = sizeof(void *) * num_possible_nodes();
> +	__uv_hub_info_list = kzalloc(bytes, GFP_KERNEL);
> +	WARN_ON_ONCE(!__uv_hub_info_list);
> +	if (!__uv_hub_info_list)
> +		return;
> +
> +	for_each_node(nodeid) {
> +		__uv_hub_info_list[nodeid] = uv_hub_info_list_blade[uv_node_to_blade_id(nodeid)];
> +	}
> +
>  	/* Initialize per CPU info: */
>  	for_each_possible_cpu(cpu) {
> -		int apicid = per_cpu(x86_cpu_to_apicid, cpu);
> -		int numa_node_id;
> +		int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
> +		unsigned short bid;
>  		unsigned short pnode;
>  
> -		nodeid = cpu_to_node(cpu);
> -		numa_node_id = numa_cpu_node(cpu);
>  		pnode = uv_apicid_to_pnode(apicid);
> +		bid = uv_pnode_to_socket(pnode) - _min_socket;
>  
> -		uv_cpu_info_per(cpu)->p_uv_hub_info = uv_hub_info_list(nodeid);
> +		uv_cpu_info_per(cpu)->p_uv_hub_info = uv_hub_info_list_blade[bid];
>  		uv_cpu_info_per(cpu)->blade_cpu_id = uv_cpu_hub_info(cpu)->nr_possible_cpus++;
>  		if (uv_cpu_hub_info(cpu)->memory_nid == NUMA_NO_NODE)
>  			uv_cpu_hub_info(cpu)->memory_nid = cpu_to_node(cpu);
>  
> -		/* Init memoryless node: */
> -		if (nodeid != numa_node_id &&
> -		    uv_hub_info_list(numa_node_id)->pnode == 0xffff)
> -			uv_hub_info_list(numa_node_id)->pnode = pnode;
> -		else if (uv_cpu_hub_info(cpu)->pnode == 0xffff)
> +		if (uv_cpu_hub_info(cpu)->pnode == 0xffff)
>  			uv_cpu_hub_info(cpu)->pnode = pnode;
>  	}
>  
> -	for_each_node(nodeid) {
> -		unsigned short pnode = uv_hub_info_list(nodeid)->pnode;
> +	for_each_possible_blade(bid) {
> +		unsigned short pnode = uv_hub_info_list_blade[bid]->pnode;
>  
>  		/* Add pnode info for pre-GAM list nodes without CPUs: */
>  		if (pnode == 0xffff) {
>  			unsigned long paddr;
>  
> -			paddr = node_start_pfn(nodeid) << PAGE_SHIFT;
> +			paddr = node_start_pfn(uv_blade_to_node(bid)) << PAGE_SHIFT;
>  			pnode = uv_gpa_to_pnode(uv_soc_phys_ram_to_gpa(paddr));
> -			uv_hub_info_list(nodeid)->pnode = pnode;
> +			uv_hub_info_list_blade[bid]->pnode = pnode;
>  		}
>  		min_pnode = min(pnode, min_pnode);
>  		max_pnode = max(pnode, max_pnode);
> -		pr_info("UV: UVHUB node:%2d pn:%02x nrcpus:%d\n",
> -			nodeid,
> -			uv_hub_info_list(nodeid)->pnode,
> -			uv_hub_info_list(nodeid)->nr_possible_cpus);
> +		pr_info("UV: HUB:%2d pn:%02x nrcpus:%d\n",
> +			bid,
> +			uv_hub_info_list_blade[bid]->pnode,
> +			uv_hub_info_list_blade[bid]->nr_possible_cpus);
>  	}
>  
>  	pr_info("UV: min_pnode:%02x max_pnode:%02x\n", min_pnode, max_pnode);
> @@ -1811,6 +1841,9 @@ static void __init uv_system_init_hub(void)
>  	map_mmr_high(max_pnode);
>  	map_mmioh_high(min_pnode, max_pnode);
>  
> +	kfree(uv_hub_info_list_blade);
> +	uv_hub_info_list_blade = NULL;
> +
>  	uv_nmi_setup();
>  	uv_cpu_init();
>  	uv_setup_proc_files(0);
> -- 
> 2.26.2
> 

-- 
Steve Wahl, Hewlett Packard Enterprise

Powered by blists - more mailing lists