| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <56608a4d-aa7e-70d0-e960-943ea9e70525@nvidia.com>
Date: Sun, 9 Oct 2022 11:14:30 +0530
From: Vidya Sagar <vidyas@...dia.com>
To: Bjorn Helgaas <helgaas@...nel.org>
Cc: bhelgaas@...gle.com, lorenzo.pieralisi@....com,
refactormyself@...il.com, kw@...ux.com, rajatja@...gle.com,
kenny@...ix.com, kai.heng.feng@...onical.com, abhsahu@...dia.com,
sagupta@...dia.com, treding@...dia.com, jonathanh@...dia.com,
linux-pci@...r.kernel.org, linux-kernel@...r.kernel.org,
kthota@...dia.com, mmaddireddy@...dia.com, sagar.tv@...il.com,
Krishna chaitanya chundru <quic_krichai@...cinc.com>
Subject: Re: [PATCH V4 1/2] PCI/ASPM: Refactor ASPM L1SS control register
programming
On 10/5/2022 8:05 AM, Bjorn Helgaas wrote:
> External email: Use caution opening links or attachments
>
>
> On Mon, Oct 03, 2022 at 12:12:22PM +0530, Vidya Sagar wrote:
>> On 9/30/2022 3:30 AM, Bjorn Helgaas wrote:
>>> On Tue, Sep 13, 2022 at 06:48:21PM +0530, Vidya Sagar wrote:
>>>> Refactor the code to extract the command code out to program
>>>> Control Registers-1 & 2 of L1 Sub-States capability to a new function
>>>> aspm_program_l1ss() and call it for both parent and child devices.
>>>>
>>>> Signed-off-by: Vidya Sagar <vidyas@...dia.com>
>>>> ---
>>>> V4:
>>>> * New patch in this series
>>>>
>>>> drivers/pci/pcie/aspm.c | 63 +++++++++++++++++++----------------------
>>>> 1 file changed, 29 insertions(+), 34 deletions(-)
>>>>
>>>> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
>>>> index a8aec190986c..ecbe3af4188d 100644
>>>> --- a/drivers/pci/pcie/aspm.c
>>>> +++ b/drivers/pci/pcie/aspm.c
>>>> @@ -455,6 +455,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
>>>> pci_write_config_dword(pdev, pos, val);
>>>> }
>>>>
>>>> +static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2)
>>>> +{
>>>> + u16 l1ss = dev->l1ss;
>>>> + u32 l1_2_enable;
>>>> +
>>>> + /*
>>>> + * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be
>>>> + * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1.
>>>> + */
>>>> + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2);
>>>> +
>>>> + /*
>>>> + * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in
>>>> + * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2
>>>> + * enable bits, even though they're all in PCI_L1SS_CTL1.
>>>> + */
>>>> + l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK;
>>>> + ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;
>>>> +
>>>> + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1);
>>>> + if (l1_2_enable)
>>>> + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1,
>>>> + ctl1 | l1_2_enable);
>>>> +}
>>>> +
>>>> /* Calculate L1.2 PM substate timing parameters */
>>>> static void aspm_calc_l1ss_info(struct pcie_link_state *link,
>>>> u32 parent_l1ss_cap, u32 child_l1ss_cap)
>>>> @@ -464,7 +489,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
>>>> u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
>>>> u32 ctl1 = 0, ctl2 = 0;
>>>> u32 pctl1, pctl2, cctl1, cctl2;
>>>> - u32 pl1_2_enables, cl1_2_enables;
>>>>
>>>> if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
>>>> return;
>>>> @@ -513,39 +537,10 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
>>>> ctl2 == pctl2 && ctl2 == cctl2)
>>>> return;
>>>>
>>>> - /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
>>>> - pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
>>>> - cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
>>>> -
>>>> - if (pl1_2_enables || cl1_2_enables) {
>>>> - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
>>>> - PCI_L1SS_CTL1_L1_2_MASK, 0);
>>>> - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
>>>> - PCI_L1SS_CTL1_L1_2_MASK, 0);
>>>> - }
>>>> -
>>>> - /* Program T_POWER_ON times in both ports */
>>>> - pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
>>>> - pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
>>>> -
>>>> - /* Program Common_Mode_Restore_Time in upstream device */
>>>> - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
>>>> - PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
>>>> -
>>>> - /* Program LTR_L1.2_THRESHOLD time in both ports */
>>>> - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
>>>> - PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>>>> - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
>>>> - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
>>>> - PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>>>> - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
>>>> -
>>>> - if (pl1_2_enables || cl1_2_enables) {
>>>> - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
>>>> - pl1_2_enables);
>>>> - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
>>>> - cl1_2_enables);
>>>> - }
>>>> + aspm_program_l1ss(parent,
>>>> + ctl1 | (pctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
>>>> + aspm_program_l1ss(child,
>>>> + ctl1 | (cctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
>>>
>>> This doesn't seem right to me. I think the intent is to update
>>> LTR_L1.2_THRESHOLD and Common_Mode_Restore_Time, which are encoded in
>>> "ctl1". It does do that, but it looks like it *also* clears
>>> everything except PCI_L1SS_CTL1_L1_2_MASK, i.e., the L1.1 Enable bits,
>>> the Link Activation bits, and the RsvdP bits, which I don't think we
>>> should be clearing. Am I missing something?
>>
>> Agree. Instead of updating some of the register fields with new values while
>> keeping other fields intact, this code programs the register fields with
>> only new values and also programming all other fields to zero which is
>> wrong.
>> Thanks for catching this. I missed it as the card with which I had tested
>> didn't have L1.1 support.
>
>> I think the following modification should fix this issue.
>>
>>
>> @@ -537,10 +537,23 @@ static void aspm_calc_l1ss_info(struct pcie_link_state
>> *link,
>> ctl2 == pctl2 && ctl2 == cctl2)
>> return;
>>
>> - aspm_program_l1ss(parent,
>> - ctl1 | (pctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
>> - aspm_program_l1ss(child,
>> - ctl1 | (cctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
>> + pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
>> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
>> + pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
>> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)
>> + );
>> + aspm_program_l1ss(parent, pctl1, ctl2);
>> +
>> + cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
>> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
>> + cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
>> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
>> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)
>> + );
>> + aspm_program_l1ss(child, cctl1, ctl2);
>> }
>>
>> static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
>>
>> But, given that you mentioned we shouldn't be touching Rsvd also, I'm
>> wondering if the following can cause any issue?
>> With the top of the tree code, the CMRT in ctrl1 register is updated only
>> for the upstream device whereas with my change, it gets updated even for the
>> downstream device. Although spec says that it is Rsvd for a downstream
>> device (i.e. upstream port), I'm wondering if we should really avoid
>> touching it?
>
> I'm not sure it's worth bothering about at this point. It feels a
> little OCD right now.
>
>> If the answer is yes, I think it is better to drop the modifications done to
>> aspm_calc_l1ss_info() function and just proceed with rest of the
>> modifications given the way it is differentiating between upstream and
>> downstream devices while updating the registers.
>> What are your comments on this?
>
> I thought we had some indication that Common_Mode_Restore_Time and
> LTR_L1.2_THRESHOLD should be programmed *before* setting the L1.2
> enable bits, even though they're all in PCI_L1SS_CTL1.
>
> The first place that comment appears is
> https://lore.kernel.org/linux-pci/20220907210540.GA140988@bhelgaas/,
> but I can't remember if it actually solved a problem or if it was just
> more OCD reading of the spec, which says "This field must only be
> modified when the ASPM L1.2 Enable bit is Clear".
>
> I'm inclined to keep the aspm_program_l1ss() changes unless we think
> they're risky, because I think it is a small step forward, at least in
> terms of reducing the number of config accesses.
>
> On pci/aspm, I currently have your v4 patches plus these tweaks:
Thanks Bjorn for pushing these modifications and I'm fine with the below
changes.
>
> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
> index dc2e21c7a9d4..016d222b07c7 100644
> --- a/drivers/pci/pcie/aspm.c
> +++ b/drivers/pci/pcie/aspm.c
> @@ -537,10 +537,21 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
> ctl2 == pctl2 && ctl2 == cctl2)
> return;
>
> - aspm_program_l1ss(parent,
> - ctl1 | (pctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
> - aspm_program_l1ss(child,
> - ctl1 | (cctl1 & PCI_L1SS_CTL1_L1_2_MASK), ctl2);
> + pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
> + pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
> + aspm_program_l1ss(parent, pctl1, ctl2);
> +
> + cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
> + cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
> + PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
> + PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
> + aspm_program_l1ss(child, cctl1, ctl2);
> }
>
> static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
> @@ -727,9 +738,6 @@ void pci_save_aspm_l1ss_state(struct pci_dev *dev)
> u16 l1ss = dev->l1ss;
> u32 *cap;
>
> - if (!pci_is_pcie(dev))
> - return;
> -
> if (!l1ss)
> return;
>
> @@ -748,9 +756,6 @@ void pci_restore_aspm_l1ss_state(struct pci_dev *dev)
> u32 *cap, ctl1, ctl2;
> u16 l1ss = dev->l1ss;
>
> - if (!pci_is_pcie(dev))
> - return;
> -
> if (!l1ss)
> return;
>
Powered by blists - more mailing lists