Skip to content

Commit ffc1138

Browse files
asraf-khanmonrog2takishidapsureshbHarinadh-Saladi
authored
Added new script validation for CSCwt38698 (#384)
* Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass <detlef.sass@de.bosch.com> * Added validation for CSCwd40071 (#332) * Added validation for CSCwd40071 * Addressed the comments * Added cversion for the check * Removed empty spaces * logic change. removed 0.0.0.0/0 and made pytest changes * logic modified and validation.md file updated * Added check for CFD - CSCwt38698 * Removed serial Number from header * Updated check name and added post upgrade check * Updated review comments * Updated review comments * Updated review comments on combining check on existing function * Resolved conflicts * Updated as review comments * Updated as review comments * Updated as review comments * Updated as review comments * Updated as review comments --------- Co-authored-by: GM <monrog2@gmail.com> Co-authored-by: Gabriel <gmonroy@cisco.com> Co-authored-by: tkishida <tkishida@cisco.com> Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb <psureshb@cisco.com> Co-authored-by: Harinadh-Saladi <hsaladi@cisco.com> Co-authored-by: sanjanch <sanjanch@cisco.com> Co-authored-by: Thatleft <detlef.sass@web.de> Co-authored-by: Detlef Sass <detlef.sass@de.bosch.com>
1 parent ea8f62e commit ffc1138

7 files changed

Lines changed: 428 additions & 29 deletions

‎aci-preupgrade-validation-script.py‎

Lines changed: 104 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2611,19 +2611,35 @@ def hw_program_fail_check(cversion, **kwargs):
26112611

26122612

26132613
@check_wrapper(check_title="Switch SSD Health (F3073, F3074 equipment-flash-warning)")
2614-
def switch_ssd_check(**kwargs):
2614+
def switch_ssd_check(cversion, tversion, **kwargs):
26152615
result = FAIL_O
2616-
headers = ["Fault", "Pod", "Node", "SSD Model", "% Threshold Crossed", "Recommended Action"]
2616+
headers = ["Fault", "Pod", "Node", "SSD Model", "% Threshold Crossed"]
26172617
data = []
2618-
unformatted_headers = ["Fault", "Fault DN", "% Threshold Crossed", "Recommended Action"]
2618+
unformatted_headers = ["Fault", "Fault DN", "% Threshold Crossed"]
26192619
unformatted_data = []
26202620
thresh = {'F3073': '90%', 'F3074': '80%'}
2621-
recommended_action = {
2622-
'F3073': 'Contact Cisco TAC for replacement procedure',
2623-
'F3074': 'Monitor (no impact to upgrades)'
2624-
}
2621+
overall_ra = ""
2622+
micron_ra = (
2623+
'\n\tRun the SSD Lifetime Validation script manually on all identified nodes before upgrading.\n'
2624+
'\tScript location: https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation\n'
2625+
)
2626+
fault_ra = "Contact Cisco TAC for replacement procedure"
2627+
mixed_ra = (
2628+
"Mixed SSD faults detected:"
2629+
"\n\tFor non-Micron SSDs (F3073/F3074 rows): Contact Cisco TAC for replacement procedure.\n"
2630+
"\tFor Micron SSD: Run the SSD Lifetime Validation script manually on all identified nodes before upgrading.\n"
2631+
"\tScript location: https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation\n"
2632+
)
2633+
26252634
doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-ssd-health"
26262635

2636+
if not tversion:
2637+
return Result(result=MANUAL, msg=TVER_MISSING)
2638+
2639+
affected = ['6.1(5e)', '6.2(1g)']
2640+
cver_affected = any(cversion.same_as(v) for v in affected)
2641+
tver_affected = any(tversion.same_as(v) for v in affected)
2642+
26272643
cs_regex = r"model:(?P<model>\w+),"
26282644
faultInsts = icurl('class',
26292645
'faultInst.json?query-target-filter=or(eq(faultInst.code,"F3073"),eq(faultInst.code,"F3074"))')
@@ -2632,25 +2648,91 @@ def switch_ssd_check(**kwargs):
26322648
dn_array = re.search(node_regex, faultInst['faultInst']['attributes']['dn'])
26332649
cs_array = re.search(cs_regex, faultInst['faultInst']['attributes']['changeSet'])
26342650
if dn_array and cs_array:
2651+
ssd_model = cs_array.group("model")
26352652
data.append([fc, dn_array.group("pod"), dn_array.group("node"),
26362653
cs_array.group("model"),
2637-
thresh.get(fc, ''),
2638-
recommended_action.get(fc, 'Resolve the fault')])
2654+
thresh.get(fc, '')])
26392655
else:
26402656
unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'],
2641-
thresh.get(fc, ''),
2642-
recommended_action.get(fc, 'Resolve the fault')])
2643-
if not data and not unformatted_data:
2644-
result = PASS
2645-
return Result(
2646-
result=result,
2647-
headers=headers,
2648-
data=data,
2649-
unformatted_headers=unformatted_headers,
2650-
unformatted_data=unformatted_data,
2651-
doc_url=doc_url,
2652-
)
2657+
thresh.get(fc, '')])
26532658

2659+
has_fault_data = bool(data or unformatted_data)
2660+
2661+
def collect_micron(classify):
2662+
eqptFlashs = icurl('class', 'eqptFlash.json?query-target-filter=eq(eqptFlash.vendor,"Micron")')
2663+
if not eqptFlashs:
2664+
return False, False
2665+
2666+
micron_ssds_per_node = defaultdict(set)
2667+
micron_rows = []
2668+
2669+
for eqptFlash in eqptFlashs:
2670+
attr = eqptFlash['eqptFlash']['attributes']
2671+
dn = re.search(node_regex, attr.get("dn", ""))
2672+
node_id = dn.group("node")
2673+
pod_id = dn.group("pod")
2674+
model = attr.get('model', '')
2675+
2676+
micron_ssds_per_node[node_id].add(model)
2677+
micron_rows.append(['CSCwt38698 (False Fault Micron SSD defect)',
2678+
pod_id,
2679+
node_id, model, 'N/A'])
2680+
2681+
if classify:
2682+
genuine_faults = []
2683+
micron_false_faults = []
2684+
2685+
for fault_row in data:
2686+
node_id = fault_row[2]
2687+
ssd_model = fault_row[3]
2688+
2689+
is_micron_fault = (node_id in micron_ssds_per_node and ssd_model in micron_ssds_per_node[node_id])
2690+
2691+
if not is_micron_fault:
2692+
genuine_faults.append(fault_row)
2693+
else:
2694+
for micron_row in micron_rows:
2695+
if micron_row[2] == node_id and micron_row[3] == ssd_model:
2696+
micron_false_faults.append(micron_row)
2697+
break
2698+
2699+
del data[:]
2700+
del unformatted_data[:]
2701+
data.extend(genuine_faults)
2702+
data.extend(micron_false_faults)
2703+
return bool(micron_false_faults), bool(genuine_faults)
2704+
else:
2705+
data.extend(micron_rows)
2706+
return True, False
2707+
2708+
if cver_affected:
2709+
has_micron_faults, has_genuine_fault = collect_micron(classify=True)
2710+
if has_micron_faults:
2711+
result = MANUAL
2712+
if has_genuine_fault:
2713+
overall_ra = mixed_ra
2714+
else:
2715+
overall_ra = micron_ra
2716+
elif has_fault_data:
2717+
result = FAIL_O
2718+
overall_ra = fault_ra
2719+
else:
2720+
result = PASS
2721+
elif tver_affected:
2722+
if has_fault_data:
2723+
result = FAIL_O
2724+
overall_ra = fault_ra
2725+
elif collect_micron(classify=False)[0]:
2726+
result = MANUAL
2727+
overall_ra = micron_ra
2728+
else:
2729+
result = PASS
2730+
else:
2731+
result = FAIL_O if has_fault_data else PASS
2732+
if has_fault_data:
2733+
overall_ra = fault_ra
2734+
2735+
return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=overall_ra, doc_url=doc_url)
26542736

26552737
# Connection Based Check
26562738
@check_wrapper(check_title="APIC SSD Health")
@@ -6967,4 +7049,4 @@ def main(_args=None):
69677049
msg = "Abort due to unexpected error - {}".format(e)
69687050
prints(msg)
69697051
log.error(msg, exc_info=True)
6970-
sys.exit(1)
7052+
sys.exit(1)

‎docs/docs/validations.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -810,6 +810,12 @@ See the [ACI Switch Node SSD Lifetime Explained technote][9] for more details.
810810
--- omit ---
811811
```
812812

813+
Due to [CSCwt38698][76], Micron SSDs present in the fabric may give false end-of-life failures after upgrading to 6.1(5e) or 6.2(1g).
814+
815+
To confirm if this is genuine or false alarm, run the SSD Lifetime Validation script on all nodes with identified actual failure case. If the SSD lifetime is critically low after manually running the script, you have to follow the SSD replacement procedure outlined in the field notice to ensure that the node remains available after the upgrade. To avoid this false alarm you can choose non-impacted target version.
816+
817+
- Script location: [SSD Lifetime Validation](https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation)
818+
813819

814820
### Config On APIC Connected Port
815821

@@ -2916,3 +2922,5 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [
29162922
[73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485
29172923
[74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741
29182924
[75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100
2925+
[76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698
2926+

‎tests/checks/switch_ssd_check/eqptFlash_mixed_node.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[
2+
{
3+
"eqptFlash": {
4+
"attributes": {
5+
"dn": "topology/pod-1/node-205/sys/ch/supslot-2/sup/flash",
6+
"model": "Micron_M600_MTFDDAT064MBF",
7+
"vendor": "Micron",
8+
"ser": "MSA20400892"
9+
}
10+
}
11+
}
12+
]

‎tests/checks/switch_ssd_check/eqptFlash_multi_micron.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"totalCount": "2",
3+
"imdata": [
4+
{
5+
"eqptFlash": {
6+
"attributes": {
7+
"dn": "topology/pod-1/node-205/sys/ch/supslot-1/sup/flash",
8+
"vendor": "Micron",
9+
"model": "Micron_M550_MTFDDAT256MAY"
10+
}
11+
}
12+
},
13+
{
14+
"eqptFlash": {
15+
"attributes": {
16+
"dn": "topology/pod-1/node-101/sys/ch/supslot-1/sup/flash",
17+
"vendor": "Micron",
18+
"model": "Micron_M600_MTFDDAT064MBF"
19+
}
20+
}
21+
}
22+
]
23+
}

‎tests/checks/switch_ssd_check/eqptFlash_single_micron_noFault.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"totalCount": "1",
3+
"imdata": [
4+
{
5+
"eqptFlash": {
6+
"attributes": {
7+
"dn": "topology/pod-1/node-103/sys/ch/supslot-1/sup/flash",
8+
"vendor": "Micron",
9+
"model": "MTFDDAK240MBB"
10+
}
11+
}
12+
}
13+
]
14+
}

‎tests/checks/switch_ssd_check/eqptFlash_single_micron_withFault.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"totalCount": "1",
3+
"imdata": [
4+
{
5+
"eqptFlash": {
6+
"attributes": {
7+
"dn": "topology/pod-1/node-205/sys/ch/supslot-1/sup/flash",
8+
"vendor": "Micron",
9+
"model": "Micron_M550_MTFDDAT256MAY"
10+
}
11+
}
12+
}
13+
]
14+
}

0 commit comments

Comments
 (0)