12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 */
33
34 #include "ixgbe_sw.h"
35
36 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
37 static char ixgbe_version[] = "ixgbe 1.1.7";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_init_unicst(ixgbe_t *);
69 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
70 static void ixgbe_setup_multicst(ixgbe_t *);
71 static void ixgbe_get_hw_state(ixgbe_t *);
72 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
73 static void ixgbe_get_conf(ixgbe_t *);
74 static void ixgbe_init_params(ixgbe_t *);
75 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
76 static void ixgbe_driver_link_check(ixgbe_t *);
77 static void ixgbe_sfp_check(void *);
78 static void ixgbe_overtemp_check(void *);
79 static void ixgbe_link_timer(void *);
80 static void ixgbe_local_timer(void *);
81 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
82 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
83 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
84 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
85 static boolean_t is_valid_mac_addr(uint8_t *);
86 static boolean_t ixgbe_stall_check(ixgbe_t *);
87 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
88 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
89 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
90 static int ixgbe_alloc_intrs(ixgbe_t *);
91 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
92 static int ixgbe_add_intr_handlers(ixgbe_t *);
93 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
94 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
95 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
96 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
97 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
98 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
297 };
298
299 static adapter_info_t ixgbe_X540_cap = {
300 128, /* maximum number of rx queues */
301 1, /* minimum number of rx queues */
302 128, /* default number of rx queues */
303 64, /* maximum number of rx groups */
304 1, /* minimum number of rx groups */
305 1, /* default number of rx groups */
306 128, /* maximum number of tx queues */
307 1, /* minimum number of tx queues */
308 8, /* default number of tx queues */
309 15500, /* maximum MTU size */
310 0xFF8, /* maximum interrupt throttle rate */
311 0, /* minimum interrupt throttle rate */
312 200, /* default interrupt throttle rate */
313 64, /* maximum total msix vectors */
314 16, /* maximum number of ring vectors */
315 2, /* maximum number of other vectors */
316 (IXGBE_EICR_LSC
317 | IXGBE_EICR_GPI_SDP1
318 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
319
320 (IXGBE_SDP1_GPIEN
321 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
322
323 (IXGBE_FLAG_DCA_CAPABLE
324 | IXGBE_FLAG_RSS_CAPABLE
325 | IXGBE_FLAG_VMDQ_CAPABLE
326 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
327 };
328
329 /*
330 * Module Initialization Functions.
331 */
332
333 int
334 _init(void)
335 {
336 int status;
337
338 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
339
340 status = mod_install(&ixgbe_modlinkage);
341
342 if (status != DDI_SUCCESS) {
343 mac_fini_ops(&ixgbe_dev_ops);
344 }
345
346 return (status);
347 }
348
409 }
410
411 /* Get the device instance */
412 instance = ddi_get_instance(devinfo);
413
414 /* Allocate memory for the instance data structure */
415 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
416
417 ixgbe->dip = devinfo;
418 ixgbe->instance = instance;
419
420 hw = &ixgbe->hw;
421 osdep = &ixgbe->osdep;
422 hw->back = osdep;
423 osdep->ixgbe = ixgbe;
424
425 /* Attach the instance pointer to the dev_info data structure */
426 ddi_set_driver_private(devinfo, ixgbe);
427
428 /*
429 * Initialize for fma support
430 */
431 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
432 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
433 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
434 ixgbe_fm_init(ixgbe);
435 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
436
437 /*
438 * Map PCI config space registers
439 */
440 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
441 ixgbe_error(ixgbe, "Failed to map PCI configurations");
442 goto attach_fail;
443 }
444 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
445
446 /*
447 * Identify the chipset family
448 */
449 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
518 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
519 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
520 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
521 ixgbe_error(ixgbe, "sfp_taskq create failed");
522 goto attach_fail;
523 }
524 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
525
526 /*
527 * Create a taskq for over-temp
528 */
529 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
530 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
531 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
532 ixgbe_error(ixgbe, "overtemp_taskq create failed");
533 goto attach_fail;
534 }
535 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
536
537 /*
538 * Initialize driver parameters
539 */
540 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
541 ixgbe_error(ixgbe, "Failed to initialize driver settings");
542 goto attach_fail;
543 }
544
545 /*
546 * Initialize mutexes for this device.
547 * Do this before enabling the interrupt handler and
548 * register the softint to avoid the condition where
549 * interrupt handler can try using uninitialized mutex.
550 */
551 ixgbe_init_locks(ixgbe);
552 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
553
554 /*
555 * Initialize chipset hardware
556 */
557 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
558 ixgbe_error(ixgbe, "Failed to initialize adapter");
559 goto attach_fail;
560 }
561 ixgbe->link_check_complete = B_FALSE;
562 ixgbe->link_check_hrtime = gethrtime() +
563 (IXGBE_LINK_UP_TIME * 100000000ULL);
564 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
565
566 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
567 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
568 goto attach_fail;
569 }
570
571 /*
572 * Initialize statistics
573 */
574 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
575 ixgbe_error(ixgbe, "Failed to initialize statistics");
576 goto attach_fail;
577 }
578 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
579
580 /*
581 * Register the driver to the MAC
582 */
583 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
584 ixgbe_error(ixgbe, "Failed to register MAC");
585 goto attach_fail;
586 }
587 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
588 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
589
590 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
591 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
592 if (ixgbe->periodic_id == 0) {
593 ixgbe_error(ixgbe, "Failed to add the link check timer");
594 goto attach_fail;
595 }
596 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
597
598 /*
599 * Now that mutex locks are initialized, and the chip is also
600 * initialized, enable interrupts.
601 */
602 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
603 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
604 goto attach_fail;
605 }
606 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
607
608 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
609 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
610
611 return (DDI_SUCCESS);
612
613 attach_fail:
614 ixgbe_unconfigure(devinfo, ixgbe);
615 return (DDI_FAILURE);
616 }
617
618 /*
619 * ixgbe_detach - Driver detach.
620 *
621 * The detach() function is the complement of the attach routine.
622 * If cmd is set to DDI_DETACH, detach() is used to remove the
623 * state associated with a given instance of a device node
624 * prior to the removal of that instance from the system.
625 *
626 * The detach() function will be called once for each instance
627 * of the device for which there has been a successful attach()
628 * once there are no longer any opens on the device.
771 */
772 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
773 ixgbe_rem_intr_handlers(ixgbe);
774 }
775
776 /*
777 * Remove taskq for sfp-status-change
778 */
779 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
780 ddi_taskq_destroy(ixgbe->sfp_taskq);
781 }
782
783 /*
784 * Remove taskq for over-temp
785 */
786 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
787 ddi_taskq_destroy(ixgbe->overtemp_taskq);
788 }
789
790 /*
791 * Remove interrupts
792 */
793 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
794 ixgbe_rem_intrs(ixgbe);
795 }
796
797 /*
798 * Unregister interrupt callback handler
799 */
800 (void) ddi_cb_unregister(ixgbe->cb_hdl);
801
802 /*
803 * Remove driver properties
804 */
805 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
806 (void) ddi_prop_remove_all(devinfo);
807 }
808
809 /*
810 * Stop the chipset
940 case ixgbe_mac_82599EB:
941 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
942 ixgbe->capab = &ixgbe_82599eb_cap;
943
944 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
945 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
946 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
947 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
948 }
949 break;
950
951 case ixgbe_mac_X540:
952 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
953 ixgbe->capab = &ixgbe_X540_cap;
954 /*
955 * For now, X540 is all set in its capab structure.
956 * As other X540 variants show up, things can change here.
957 */
958 break;
959
960 default:
961 IXGBE_DEBUGLOG_1(ixgbe,
962 "adapter not supported in ixgbe_identify_hardware(): %d\n",
963 hw->mac.type);
964 return (IXGBE_FAILURE);
965 }
966
967 return (IXGBE_SUCCESS);
968 }
969
970 /*
971 * ixgbe_regs_map - Map the device registers.
972 *
973 */
974 static int
975 ixgbe_regs_map(ixgbe_t *ixgbe)
976 {
977 dev_info_t *devinfo = ixgbe->dip;
978 struct ixgbe_hw *hw = &ixgbe->hw;
979 struct ixgbe_osdep *osdep = &ixgbe->osdep;
994 (caddr_t *)&hw->hw_addr, 0,
995 mem_size, &ixgbe_regs_acc_attr,
996 &osdep->reg_handle)) != DDI_SUCCESS) {
997 return (IXGBE_FAILURE);
998 }
999
1000 return (IXGBE_SUCCESS);
1001 }
1002
1003 /*
1004 * ixgbe_init_properties - Initialize driver properties.
1005 */
1006 static void
1007 ixgbe_init_properties(ixgbe_t *ixgbe)
1008 {
1009 /*
1010 * Get conf file properties, including link settings
1011 * jumbo frames, ring number, descriptor number, etc.
1012 */
1013 ixgbe_get_conf(ixgbe);
1014
1015 ixgbe_init_params(ixgbe);
1016 }
1017
1018 /*
1019 * ixgbe_init_driver_settings - Initialize driver settings.
1020 *
1021 * The settings include hardware function pointers, bus information,
1022 * rx/tx rings settings, link state, and any other parameters that
1023 * need to be setup during driver initialization.
1024 */
1025 static int
1026 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1027 {
1028 struct ixgbe_hw *hw = &ixgbe->hw;
1029 dev_info_t *devinfo = ixgbe->dip;
1030 ixgbe_rx_ring_t *rx_ring;
1031 ixgbe_rx_group_t *rx_group;
1032 ixgbe_tx_ring_t *tx_ring;
1033 uint32_t rx_size;
1034 uint32_t tx_size;
1035 uint32_t ring_per_group;
1230 ixgbe_stop(ixgbe, B_FALSE);
1231
1232 mutex_exit(&ixgbe->gen_lock);
1233
1234 /*
1235 * Disable and stop the watchdog timer
1236 */
1237 ixgbe_disable_watchdog_timer(ixgbe);
1238
1239 return (DDI_SUCCESS);
1240 }
1241
1242 /*
1243 * ixgbe_init - Initialize the device.
1244 */
1245 static int
1246 ixgbe_init(ixgbe_t *ixgbe)
1247 {
1248 struct ixgbe_hw *hw = &ixgbe->hw;
1249 u8 pbanum[IXGBE_PBANUM_LENGTH];
1250
1251 mutex_enter(&ixgbe->gen_lock);
1252
1253 /*
1254 * Reset chipset to put the hardware in a known state
1255 * before we try to do anything with the eeprom.
1256 */
1257 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1258 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1259 goto init_fail;
1260 }
1261
1262 /*
1263 * Need to init eeprom before validating the checksum.
1264 */
1265 if (ixgbe_init_eeprom_params(hw) < 0) {
1266 ixgbe_error(ixgbe,
1267 "Unable to intitialize the eeprom interface.");
1268 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1269 goto init_fail;
1270 }
1271
1272 /*
1273 * NVM validation
1274 */
1275 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1276 /*
1277 * Some PCI-E parts fail the first check due to
1278 * the link being in sleep state. Call it again,
1279 * if it fails a second time it's a real issue.
1280 */
1281 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1282 ixgbe_error(ixgbe,
1283 "Invalid NVM checksum. Please contact "
1284 "the vendor to update the NVM.");
1285 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1286 goto init_fail;
1287 }
1288 }
1289
1290 /*
1291 * Setup default flow control thresholds - enable/disable
1292 * & flow control type is controlled by ixgbe.conf
1293 */
1294 hw->fc.high_water[0] = DEFAULT_FCRTH;
1295 hw->fc.low_water[0] = DEFAULT_FCRTL;
1296 hw->fc.pause_time = DEFAULT_FCPAUSE;
1297 hw->fc.send_xon = B_TRUE;
1298
1299 /*
1300 * Initialize link settings
1301 */
1302 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1303
1304 /*
1305 * Initialize the chipset hardware
1306 */
1307 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1308 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1309 goto init_fail;
1310 }
1311
1312 /*
1313 * Read identifying information and place in devinfo.
1314 */
1315 pbanum[0] = '\0';
1316 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1317 if (*pbanum != '\0') {
1318 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1319 "printed-board-assembly", (char *)pbanum);
1327 return (IXGBE_SUCCESS);
1328
1329 init_fail:
1330 /*
1331 * Reset PHY
1332 */
1333 (void) ixgbe_reset_phy(hw);
1334
1335 mutex_exit(&ixgbe->gen_lock);
1336 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1337 return (IXGBE_FAILURE);
1338 }
1339
1340 /*
1341 * ixgbe_chip_start - Initialize and start the chipset hardware.
1342 */
1343 static int
1344 ixgbe_chip_start(ixgbe_t *ixgbe)
1345 {
1346 struct ixgbe_hw *hw = &ixgbe->hw;
1347 int ret_val, i;
1348
1349 ASSERT(mutex_owned(&ixgbe->gen_lock));
1350
1351 /*
1352 * Get the mac address
1353 * This function should handle SPARC case correctly.
1354 */
1355 if (!ixgbe_find_mac_address(ixgbe)) {
1356 ixgbe_error(ixgbe, "Failed to get the mac address");
1357 return (IXGBE_FAILURE);
1358 }
1359
1360 /*
1361 * Validate the mac address
1362 */
1363 (void) ixgbe_init_rx_addrs(hw);
1364 if (!is_valid_mac_addr(hw->mac.addr)) {
1365 ixgbe_error(ixgbe, "Invalid mac address");
1366 return (IXGBE_FAILURE);
1367 }
1368
1369 /*
1370 * Configure/Initialize hardware
1371 */
1372 ret_val = ixgbe_init_hw(hw);
1373 if (ret_val != IXGBE_SUCCESS) {
1374 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1375 ixgbe_error(ixgbe,
1376 "This 82599 device is pre-release and contains"
1377 " outdated firmware, please contact your hardware"
1378 " vendor for a replacement.");
1379 } else {
1380 ixgbe_error(ixgbe, "Failed to initialize hardware");
1381 return (IXGBE_FAILURE);
1382 }
1383 }
1384
1385 /*
1386 * Re-enable relaxed ordering for performance. It is disabled
1387 * by default in the hardware init.
1388 */
1389 if (ixgbe->relax_order_enable == B_TRUE)
1390 ixgbe_enable_relaxed_ordering(hw);
1391
1392 /*
1393 * Setup adapter interrupt vectors
1394 */
1395 ixgbe_setup_adapter_vector(ixgbe);
1396
1397 /*
1398 * Initialize unicast addresses.
1399 */
1400 ixgbe_init_unicst(ixgbe);
1401
1402 /*
1403 * Setup and initialize the mctable structures.
1404 */
1405 ixgbe_setup_multicst(ixgbe);
1406
1407 /*
1408 * Set interrupt throttling rate
1409 */
1410 for (i = 0; i < ixgbe->intr_cnt; i++) {
1411 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1412 }
1413
1414 /*
1415 * Save the state of the phy
1416 */
1417 ixgbe_get_hw_state(ixgbe);
1418
1419 /*
1420 * Make sure driver has control
1421 */
1422 ixgbe_get_driver_control(hw);
1423
1424 return (IXGBE_SUCCESS);
1425 }
1426
1427 /*
1428 * ixgbe_chip_stop - Stop the chipset hardware
1429 */
1430 static void
1431 ixgbe_chip_stop(ixgbe_t *ixgbe)
1432 {
1433 struct ixgbe_hw *hw = &ixgbe->hw;
1434
1435 ASSERT(mutex_owned(&ixgbe->gen_lock));
1436
1437 /*
1438 * Tell firmware driver is no longer in control
1439 */
1440 ixgbe_release_driver_control(hw);
1441
1442 /*
1443 * Reset the chipset
1444 */
1445 (void) ixgbe_reset_hw(hw);
1446
1447 /*
1448 * Reset PHY
1449 */
1450 (void) ixgbe_reset_phy(hw);
1451 }
1452
1453 /*
1454 * ixgbe_reset - Reset the chipset and re-start the driver.
1455 *
1456 * It involves stopping and re-starting the chipset,
1457 * and re-configuring the rx/tx rings.
1458 */
1459 static int
1460 ixgbe_reset(ixgbe_t *ixgbe)
1461 {
1462 int i;
1463
1464 /*
1465 * Disable and stop the watchdog timer
1466 */
1467 ixgbe_disable_watchdog_timer(ixgbe);
1468
1469 mutex_enter(&ixgbe->gen_lock);
1470
1632 * Otherwise return B_FALSE;
1633 */
1634 for (i = 0; i < RX_DRAIN_TIME; i++) {
1635 done = (ixgbe->rcb_pending == 0);
1636
1637 if (done)
1638 break;
1639
1640 msec_delay(1);
1641 }
1642
1643 return (done);
1644 }
1645
1646 /*
1647 * ixgbe_start - Start the driver/chipset.
1648 */
1649 int
1650 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1651 {
1652 int i;
1653
1654 ASSERT(mutex_owned(&ixgbe->gen_lock));
1655
1656 if (alloc_buffer) {
1657 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1658 ixgbe_error(ixgbe,
1659 "Failed to allocate software receive rings");
1660 return (IXGBE_FAILURE);
1661 }
1662
1663 /* Allocate buffers for all the rx/tx rings */
1664 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1665 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1666 return (IXGBE_FAILURE);
1667 }
1668
1669 ixgbe->tx_ring_init = B_TRUE;
1670 } else {
1671 ixgbe->tx_ring_init = B_FALSE;
1672 }
1673
1674 for (i = 0; i < ixgbe->num_rx_rings; i++)
1675 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1676 for (i = 0; i < ixgbe->num_tx_rings; i++)
1677 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1678
1679 /*
1680 * Start the chipset hardware
1681 */
1682 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1683 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1684 goto start_failure;
1685 }
1686
1687 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1688 goto start_failure;
1689 }
1690
1691 /*
1692 * Setup the rx/tx rings
1693 */
1694 ixgbe_setup_rings(ixgbe);
1695
1696 /*
1697 * ixgbe_start() will be called when resetting, however if reset
1698 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1699 * before enabling the interrupts.
1700 */
1701 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1702 | IXGBE_STALL| IXGBE_OVERTEMP));
1703
1704 /*
1705 * Enable adapter interrupts
1706 * The interrupts must be enabled after the driver state is START
2174
2175 /*
2176 * Setup head & tail pointers
2177 */
2178 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2179 rx_data->ring_size - 1);
2180 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2181
2182 rx_data->rbd_next = 0;
2183 rx_data->lro_first = 0;
2184
2185 /*
2186 * Setup the Receive Descriptor Control Register (RXDCTL)
2187 * PTHRESH=32 descriptors (half the internal cache)
2188 * HTHRESH=0 descriptors (to minimize latency on fetch)
2189 * WTHRESH defaults to 1 (writeback each descriptor)
2190 */
2191 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2192 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2193
2194 /* Not a valid value for 82599 or X540 */
2195 if (hw->mac.type == ixgbe_mac_82598EB) {
2196 reg_val |= 0x0020; /* pthresh */
2197 }
2198 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2199
2200 if (hw->mac.type == ixgbe_mac_82599EB ||
2201 hw->mac.type == ixgbe_mac_X540) {
2202 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2203 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2204 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2205 }
2206
2207 /*
2208 * Setup the Split and Replication Receive Control Register.
2209 * Set the rx buffer size and the advanced descriptor type.
2210 */
2211 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2212 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2213 reg_val |= IXGBE_SRRCTL_DROP_EN;
2214 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2215 }
2216
2217 static void
2218 ixgbe_setup_rx(ixgbe_t *ixgbe)
2219 {
2220 ixgbe_rx_ring_t *rx_ring;
2221 struct ixgbe_hw *hw = &ixgbe->hw;
2222 uint32_t reg_val;
2223 uint32_t ring_mapping;
2224 uint32_t i, index;
2225 uint32_t psrtype_rss_bit;
2226
2227 /* PSRTYPE must be configured for 82599 */
2228 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2229 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2230 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2231 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2232 reg_val |= IXGBE_PSRTYPE_L2HDR;
2233 reg_val |= 0x80000000;
2234 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2235 } else {
2236 if (ixgbe->num_rx_groups > 32) {
2237 psrtype_rss_bit = 0x20000000;
2238 } else {
2239 psrtype_rss_bit = 0x40000000;
2240 }
2241 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2242 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2243 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2244 reg_val |= IXGBE_PSRTYPE_L2HDR;
2245 reg_val |= psrtype_rss_bit;
2246 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2247 }
2248 }
2249
2250 /*
2251 * Set filter control in FCTRL to accept broadcast packets and do
2252 * not pass pause frames to host. Flow control settings are already
2253 * in this register, so preserve them.
2254 */
2255 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2256 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2257 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2258 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2259
2260 /*
2261 * Hardware checksum settings
2262 */
2263 if (ixgbe->rx_hcksum_enable) {
2264 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2265 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2266 }
2267
2268 /*
2269 * Setup VMDq and RSS for multiple receive queues
2270 */
2271 switch (ixgbe->classify_mode) {
2272 case IXGBE_CLASSIFY_RSS:
2273 /*
2274 * One group, only RSS is needed when more than
2275 * one ring enabled.
2276 */
2277 ixgbe_setup_rss(ixgbe);
2278 break;
2279
2280 case IXGBE_CLASSIFY_VMDQ:
2281 /*
2282 * Multiple groups, each group has one ring,
2283 * only VMDq is needed.
2284 */
2285 ixgbe_setup_vmdq(ixgbe);
2286 break;
2287
2288 case IXGBE_CLASSIFY_VMDQ_RSS:
2289 /*
2290 * Multiple groups and multiple rings, both
2291 * VMDq and RSS are needed.
2292 */
2293 ixgbe_setup_vmdq_rss(ixgbe);
2294 break;
2295
2296 default:
2297 break;
2298 }
2299
2300 /*
2301 * Enable the receive unit. This must be done after filter
2302 * control is set in FCTRL.
2303 */
2304 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2305 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2306 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2307
2308 /*
2309 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2310 */
2311 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2312 rx_ring = &ixgbe->rx_rings[i];
2313 ixgbe_setup_rx_ring(rx_ring);
2314 }
2315
2316 /*
2317 * Setup the per-ring statistics mapping.
2318 */
2319 ring_mapping = 0;
2320 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2321 index = ixgbe->rx_rings[i].hw_index;
2322 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2323 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2324 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2325 }
2326
2327 /*
2328 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2329 * by four bytes if the packet has a VLAN field, so includes MTU,
2330 * ethernet header and frame check sequence.
2331 * Register is MAXFRS in 82599.
2332 */
2333 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2334 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2335 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2336
2337 /*
2338 * Setup Jumbo Frame enable bit
2339 */
2340 if (ixgbe->default_mtu > ETHERMTU) {
2341 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2342 reg_val |= IXGBE_HLREG0_JUMBOEN;
2343 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2344 }
2345
2346 /*
2347 * Setup RSC for multiple receive queues.
2348 */
2349 if (ixgbe->lro_enable) {
2350 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2351 /*
2352 * Make sure rx_buf_size * MAXDESC not greater
2353 * than 65535.
2354 * Intel recommends 4 for MAXDESC field value.
2355 */
2356 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2357 reg_val |= IXGBE_RSCCTL_RSCEN;
2358 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2359 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2360 else
2361 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2362 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2363 }
2364
2479 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2480 tx_ring = &ixgbe->tx_rings[i];
2481 ixgbe_setup_tx_ring(tx_ring);
2482 }
2483
2484 /*
2485 * Setup the per-ring statistics mapping.
2486 */
2487 ring_mapping = 0;
2488 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2489 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2490 if ((i & 0x3) == 0x3) {
2491 switch (hw->mac.type) {
2492 case ixgbe_mac_82598EB:
2493 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2494 ring_mapping);
2495 break;
2496
2497 case ixgbe_mac_82599EB:
2498 case ixgbe_mac_X540:
2499 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2500 ring_mapping);
2501 break;
2502
2503 default:
2504 break;
2505 }
2506
2507 ring_mapping = 0;
2508 }
2509 }
2510 if (i & 0x3) {
2511 switch (hw->mac.type) {
2512 case ixgbe_mac_82598EB:
2513 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2514 break;
2515
2516 case ixgbe_mac_82599EB:
2517 case ixgbe_mac_X540:
2518 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2519 break;
2520
2521 default:
2522 break;
2523 }
2524 }
2525
2526 /*
2527 * Enable CRC appending and TX padding (for short tx frames)
2528 */
2529 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2530 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2531 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2532
2533 /*
2534 * enable DMA for 82599 and X540 parts
2535 */
2536 if (hw->mac.type == ixgbe_mac_82599EB ||
2537 hw->mac.type == ixgbe_mac_X540) {
2538 /* DMATXCTL.TE must be set after all Tx config is complete */
2539 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2540 reg_val |= IXGBE_DMATXCTL_TE;
2541 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2542
2543 /* Disable arbiter to set MTQC */
2544 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2545 reg_val |= IXGBE_RTTDCS_ARBDIS;
2546 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2547 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2548 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2549 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2550 }
2551
2552 /*
2553 * Enabling tx queues ..
2554 * For 82599 must be done after DMATXCTL.TE is set
2555 */
2556 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2557 tx_ring = &ixgbe->tx_rings[i];
2558 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2559 reg_val |= IXGBE_TXDCTL_ENABLE;
2560 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2561 }
2562 }
2563
2564 /*
2565 * ixgbe_setup_rss - Setup receive-side scaling feature.
2566 */
2567 static void
2568 ixgbe_setup_rss(ixgbe_t *ixgbe)
2569 {
2570 struct ixgbe_hw *hw = &ixgbe->hw;
2571 uint32_t i, mrqc, rxcsum;
2572 uint32_t random;
2573 uint32_t reta;
2574 uint32_t ring_per_group;
2575
2576 /*
2577 * Fill out redirection table
2578 */
2579 reta = 0;
2580 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2581
2582 for (i = 0; i < 128; i++) {
2583 reta = (reta << 8) | (i % ring_per_group) |
2584 ((i % ring_per_group) << 4);
2585 if ((i & 3) == 3)
2586 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2587 }
2588
2589 /*
2590 * Fill out hash function seeds with a random constant
2591 */
2592 for (i = 0; i < 10; i++) {
2593 (void) random_get_pseudo_bytes((uint8_t *)&random,
2594 sizeof (uint32_t));
2595 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2596 }
2597
2598 /*
2599 * Enable RSS & perform hash on these packet types
2600 */
2601 mrqc = IXGBE_MRQC_RSSEN |
2602 IXGBE_MRQC_RSS_FIELD_IPV4 |
2603 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2604 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2605 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2606 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2607 IXGBE_MRQC_RSS_FIELD_IPV6 |
2608 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2609 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2610 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2611 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2612
2613 /*
2614 * Disable Packet Checksum to enable RSS for multiple receive queues.
2615 * It is an adapter hardware limitation that Packet Checksum is
2616 * mutually exclusive with RSS.
2617 */
2618 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2619 rxcsum |= IXGBE_RXCSUM_PCSD;
2620 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2621 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2622 }
2623
2624 /*
2625 * ixgbe_setup_vmdq - Setup MAC classification feature
2626 */
2627 static void
2628 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2629 {
2630 struct ixgbe_hw *hw = &ixgbe->hw;
2631 uint32_t vmdctl, i, vtctl;
2632
2633 /*
2634 * Setup the VMDq Control register, enable VMDq based on
2635 * packet destination MAC address:
2636 */
2637 switch (hw->mac.type) {
2638 case ixgbe_mac_82598EB:
2639 /*
2640 * VMDq Enable = 1;
2641 * VMDq Filter = 0; MAC filtering
2642 * Default VMDq output index = 0;
2643 */
2644 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2645 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2646 break;
2647
2648 case ixgbe_mac_82599EB:
2649 case ixgbe_mac_X540:
2650 /*
2651 * Enable VMDq-only.
2652 */
2653 vmdctl = IXGBE_MRQC_VMDQEN;
2654 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2655
2656 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2657 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2658 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2659 }
2660
2661 /*
2662 * Enable Virtualization and Replication.
2663 */
2664 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2665 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2666
2667 /*
2668 * Enable receiving packets to all VFs
2669 */
2670 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2671 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2672 break;
2673
2674 default:
2675 break;
2676 }
2677 }
2678
2679 /*
2680 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2681 */
2682 static void
2683 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2684 {
2685 struct ixgbe_hw *hw = &ixgbe->hw;
2686 uint32_t i, mrqc, rxcsum;
2687 uint32_t random;
2688 uint32_t reta;
2689 uint32_t ring_per_group;
2690 uint32_t vmdctl, vtctl;
2691
2692 /*
2693 * Fill out redirection table
2694 */
2695 reta = 0;
2696 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2697 for (i = 0; i < 128; i++) {
2698 reta = (reta << 8) | (i % ring_per_group) |
2699 ((i % ring_per_group) << 4);
2700 if ((i & 3) == 3)
2701 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2702 }
2703
2704 /*
2705 * Fill out hash function seeds with a random constant
2706 */
2707 for (i = 0; i < 10; i++) {
2708 (void) random_get_pseudo_bytes((uint8_t *)&random,
2709 sizeof (uint32_t));
2710 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2711 }
2712
2713 /*
2714 * Enable and setup RSS and VMDq
2715 */
2716 switch (hw->mac.type) {
2717 case ixgbe_mac_82598EB:
2718 /*
2719 * Enable RSS & Setup RSS Hash functions
2720 */
2721 mrqc = IXGBE_MRQC_RSSEN |
2722 IXGBE_MRQC_RSS_FIELD_IPV4 |
2723 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2724 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2725 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2726 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2727 IXGBE_MRQC_RSS_FIELD_IPV6 |
2728 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2729 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2730 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2731 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2732
2733 /*
2734 * Enable and Setup VMDq
2735 * VMDq Filter = 0; MAC filtering
2736 * Default VMDq output index = 0;
2737 */
2738 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2739 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2740 break;
2741
2742 case ixgbe_mac_82599EB:
2743 case ixgbe_mac_X540:
2744 /*
2745 * Enable RSS & Setup RSS Hash functions
2746 */
2747 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2748 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2749 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2750 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2751 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2752 IXGBE_MRQC_RSS_FIELD_IPV6 |
2753 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2754 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2755 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2756
2757 /*
2758 * Enable VMDq+RSS.
2759 */
2760 if (ixgbe->num_rx_groups > 32) {
2761 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2762 } else {
2763 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2764 }
2765
2766 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2767
2768 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2769 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2770 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2771 }
2772 break;
2773
2774 default:
2775 break;
2776
2777 }
2778
2779 /*
2780 * Disable Packet Checksum to enable RSS for multiple receive queues.
2781 * It is an adapter hardware limitation that Packet Checksum is
2782 * mutually exclusive with RSS.
2783 */
2784 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2785 rxcsum |= IXGBE_RXCSUM_PCSD;
2786 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2787 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2788
2789 if (hw->mac.type == ixgbe_mac_82599EB ||
2790 hw->mac.type == ixgbe_mac_X540) {
2791 /*
2792 * Enable Virtualization and Replication.
2793 */
2794 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2795 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2796
2797 /*
2798 * Enable receiving packets to all VFs
2799 */
2800 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2801 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2802 }
2803 }
2804
2805 /*
2806 * ixgbe_init_unicst - Initialize the unicast addresses.
2807 */
2808 static void
2809 ixgbe_init_unicst(ixgbe_t *ixgbe)
2810 {
2811 struct ixgbe_hw *hw = &ixgbe->hw;
2812 uint8_t *mac_addr;
2813 int slot;
2814 /*
2815 * Here we should consider two situations:
2816 *
2817 * 1. Chipset is initialized at the first time,
2818 * Clear all the multiple unicast addresses.
2819 *
2820 * 2. Chipset is reset
2821 * Recover the multiple unicast addresses from the
2822 * software data structure to the RAR registers.
2823 */
2824 if (!ixgbe->unicst_init) {
2825 /*
2982 /*
2983 * 82598 supports the following combination:
2984 * vmdq no. x rss no.
2985 * [5..16] x 1
2986 * [1..4] x [1..16]
2987 * However 8 rss queue per pool (vmdq) is sufficient for
2988 * most cases.
2989 */
2990 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2991 if (ixgbe->num_rx_groups > 4) {
2992 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2993 } else {
2994 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2995 min(8, ring_per_group);
2996 }
2997
2998 break;
2999
3000 case ixgbe_mac_82599EB:
3001 case ixgbe_mac_X540:
3002 /*
3003 * 82599 supports the following combination:
3004 * vmdq no. x rss no.
3005 * [33..64] x [1..2]
3006 * [2..32] x [1..4]
3007 * 1 x [1..16]
3008 * However 8 rss queue per pool (vmdq) is sufficient for
3009 * most cases.
3010 *
3011 * For now, treat X540 like the 82599.
3012 */
3013 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3014 if (ixgbe->num_rx_groups == 1) {
3015 ixgbe->num_rx_rings = min(8, ring_per_group);
3016 } else if (ixgbe->num_rx_groups <= 32) {
3017 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3018 min(4, ring_per_group);
3019 } else if (ixgbe->num_rx_groups <= 64) {
3020 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3021 min(2, ring_per_group);
3022 }
3023 break;
3024
3025 default:
3026 break;
3027 }
3028
3029 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3030
3031 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3154 * 1 = force interrupt type MSI-X
3155 * 2 = force interrupt type MSI
3156 * 3 = force interrupt type Legacy
3157 */
3158 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3159 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3160
3161 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3162 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3163 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3164 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3165 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3166 0, 1, DEFAULT_LSO_ENABLE);
3167 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3168 0, 1, DEFAULT_LRO_ENABLE);
3169 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3170 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3171 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3172 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3173
3174 /* Head Write Back not recommended for 82599 and X540 */
3175 if (hw->mac.type == ixgbe_mac_82599EB ||
3176 hw->mac.type == ixgbe_mac_X540) {
3177 ixgbe->tx_head_wb_enable = B_FALSE;
3178 }
3179
3180 /*
3181 * ixgbe LSO needs the tx h/w checksum support.
3182 * LSO will be disabled if tx h/w checksum is not
3183 * enabled.
3184 */
3185 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3186 ixgbe->lso_enable = B_FALSE;
3187 }
3188
3189 /*
3190 * ixgbe LRO needs the rx h/w checksum support.
3191 * LRO will be disabled if rx h/w checksum is not
3192 * enabled.
3193 */
3194 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3195 ixgbe->lro_enable = B_FALSE;
3196 }
3197
3198 /*
3199 * ixgbe LRO only been supported by 82599 and X540 now
3200 */
3201 if (hw->mac.type == ixgbe_mac_82598EB) {
3202 ixgbe->lro_enable = B_FALSE;
3203 }
3204 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3205 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3206 DEFAULT_TX_COPY_THRESHOLD);
3207 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3208 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3209 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3210 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3211 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3212 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3213 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3214 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3215 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3216
3217 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3218 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3219 DEFAULT_RX_COPY_THRESHOLD);
3220 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3221 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3222 DEFAULT_RX_LIMIT_PER_INTR);
3223
3224 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3225 ixgbe->capab->min_intr_throttle,
3226 ixgbe->capab->max_intr_throttle,
3227 ixgbe->capab->def_intr_throttle);
3228 /*
3229 * 82599 and X540 require the interrupt throttling rate is
3230 * a multiple of 8. This is enforced by the register
3231 * definiton.
3232 */
3233 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540)
3234 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3235
3236 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3237 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3238 }
3239
3240 static void
3241 ixgbe_init_params(ixgbe_t *ixgbe)
3242 {
3243 ixgbe->param_en_10000fdx_cap = 1;
3244 ixgbe->param_en_1000fdx_cap = 1;
3245 ixgbe->param_en_100fdx_cap = 1;
3246 ixgbe->param_adv_10000fdx_cap = 1;
3247 ixgbe->param_adv_1000fdx_cap = 1;
3248 ixgbe->param_adv_100fdx_cap = 1;
3249
3250 ixgbe->param_pause_cap = 1;
3251 ixgbe->param_asym_pause_cap = 1;
3252 ixgbe->param_rem_fault = 0;
3253
3254 ixgbe->param_adv_autoneg_cap = 1;
3255 ixgbe->param_adv_pause_cap = 1;
3256 ixgbe->param_adv_asym_pause_cap = 1;
3257 ixgbe->param_adv_rem_fault = 0;
3258
3259 ixgbe->param_lp_10000fdx_cap = 0;
3260 ixgbe->param_lp_1000fdx_cap = 0;
3261 ixgbe->param_lp_100fdx_cap = 0;
3262 ixgbe->param_lp_autoneg_cap = 0;
3263 ixgbe->param_lp_pause_cap = 0;
3264 ixgbe->param_lp_asym_pause_cap = 0;
3265 ixgbe->param_lp_rem_fault = 0;
3266 }
3267
3268 /*
3269 * ixgbe_get_prop - Get a property value out of the configuration file
3270 * ixgbe.conf.
3271 *
3272 * Caller provides the name of the property, a default value, a minimum
3273 * value, and a maximum value.
3274 *
3275 * Return configured value of the property, with default, minimum and
3276 * maximum properly applied.
3277 */
3278 static int
3279 ixgbe_get_prop(ixgbe_t *ixgbe,
3287 /*
3288 * Call ddi_prop_get_int() to read the conf settings
3289 */
3290 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3291 DDI_PROP_DONTPASS, propname, defval);
3292 if (value > maxval)
3293 value = maxval;
3294
3295 if (value < minval)
3296 value = minval;
3297
3298 return (value);
3299 }
3300
3301 /*
3302 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3303 */
3304 int
3305 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3306 {
3307 u32 autoneg_advertised = 0;
3308
3309 /*
3310 * No half duplex support with 10Gb parts
3311 */
3312 if (ixgbe->param_adv_10000fdx_cap == 1)
3313 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3314
3315 if (ixgbe->param_adv_1000fdx_cap == 1)
3316 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3317
3318 if (ixgbe->param_adv_100fdx_cap == 1)
3319 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3320
3321 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3322 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3323 "to autonegotiation with full link capabilities.");
3324
3325 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3326 IXGBE_LINK_SPEED_1GB_FULL |
3327 IXGBE_LINK_SPEED_100_FULL;
3328 }
3329
3330 if (setup_hw) {
3331 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3332 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3333 ixgbe_notice(ixgbe, "Setup link failed on this "
3334 "device.");
3335 return (IXGBE_FAILURE);
3336 }
3337 }
3338
3339 return (IXGBE_SUCCESS);
3340 }
3341
3342 /*
3343 * ixgbe_driver_link_check - Link status processing.
3344 *
3345 * This function can be called in both kernel context and interrupt context
3346 */
3347 static void
3348 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3349 {
3350 struct ixgbe_hw *hw = &ixgbe->hw;
3351 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3352 boolean_t link_up = B_FALSE;
3353 boolean_t link_changed = B_FALSE;
3354
3355 ASSERT(mutex_owned(&ixgbe->gen_lock));
3356
3357 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3358 if (link_up) {
3359 ixgbe->link_check_complete = B_TRUE;
3360
3361 /* Link is up, enable flow control settings */
3362 (void) ixgbe_fc_enable(hw);
3363
3364 /*
3365 * The Link is up, check whether it was marked as down earlier
3366 */
3367 if (ixgbe->link_state != LINK_STATE_UP) {
3368 switch (speed) {
3369 case IXGBE_LINK_SPEED_10GB_FULL:
3370 ixgbe->link_speed = SPEED_10GB;
3371 break;
3372 case IXGBE_LINK_SPEED_1GB_FULL:
3373 ixgbe->link_speed = SPEED_1GB;
3374 break;
3375 case IXGBE_LINK_SPEED_100_FULL:
3376 ixgbe->link_speed = SPEED_100;
3377 }
3378 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3379 ixgbe->link_state = LINK_STATE_UP;
3380 link_changed = B_TRUE;
3381 }
3382 } else {
3383 if (ixgbe->link_check_complete == B_TRUE ||
3384 (ixgbe->link_check_complete == B_FALSE &&
3385 gethrtime() >= ixgbe->link_check_hrtime)) {
3386 /*
3387 * The link is really down
3388 */
3389 ixgbe->link_check_complete = B_TRUE;
3390
3391 if (ixgbe->link_state != LINK_STATE_DOWN) {
3405 ixgbe->eims |= IXGBE_EICR_LSC;
3406 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3407 }
3408
3409 if (link_changed) {
3410 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3411 }
3412 }
3413
3414 /*
3415 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3416 */
3417 static void
3418 ixgbe_sfp_check(void *arg)
3419 {
3420 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3421 uint32_t eicr = ixgbe->eicr;
3422 struct ixgbe_hw *hw = &ixgbe->hw;
3423
3424 mutex_enter(&ixgbe->gen_lock);
3425 if (eicr & IXGBE_EICR_GPI_SDP1) {
3426 /* clear the interrupt */
3427 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3428
3429 /* if link up, do multispeed fiber setup */
3430 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3431 B_TRUE, B_TRUE);
3432 ixgbe_driver_link_check(ixgbe);
3433 ixgbe_get_hw_state(ixgbe);
3434 } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3435 /* clear the interrupt */
3436 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3437
3438 /* if link up, do sfp module setup */
3439 (void) hw->mac.ops.setup_sfp(hw);
3440
3441 /* do multispeed fiber setup */
3442 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3443 B_TRUE, B_TRUE);
3444 ixgbe_driver_link_check(ixgbe);
3445 ixgbe_get_hw_state(ixgbe);
3446 }
3447 mutex_exit(&ixgbe->gen_lock);
3448
3449 /*
3450 * We need to fully re-check the link later.
3451 */
3452 ixgbe->link_check_complete = B_FALSE;
3453 ixgbe->link_check_hrtime = gethrtime() +
3454 (IXGBE_LINK_UP_TIME * 100000000ULL);
3455 }
3456
3457 /*
3458 * ixgbe_overtemp_check - overtemp module processing done in taskq
3459 *
3460 * This routine will only be called on adapters with temperature sensor.
3461 * The indication of over-temperature can be either SDP0 interrupt or the link
3462 * status change interrupt.
3463 */
3464 static void
3465 ixgbe_overtemp_check(void *arg)
3466 {
3467 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3468 struct ixgbe_hw *hw = &ixgbe->hw;
3469 uint32_t eicr = ixgbe->eicr;
3470 ixgbe_link_speed speed;
3471 boolean_t link_up;
3472
3473 mutex_enter(&ixgbe->gen_lock);
3474
3475 /* make sure we know current state of link */
3476 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3477
3478 /* check over-temp condition */
3479 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3480 (eicr & IXGBE_EICR_LSC)) {
3481 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3482 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3483
3484 /*
3485 * Disable the adapter interrupts
3486 */
3487 ixgbe_disable_adapter_interrupts(ixgbe);
3488
3489 /*
3490 * Disable Rx/Tx units
3491 */
3492 (void) ixgbe_stop_adapter(hw);
3493
3494 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3495 ixgbe_error(ixgbe,
3496 "Problem: Network adapter has been stopped "
3497 "because it has overheated");
3498 ixgbe_error(ixgbe,
3499 "Action: Restart the computer. "
3500 "If the problem persists, power off the system "
3501 "and replace the adapter");
3502 }
3503 }
3504
3505 /* write to clear the interrupt */
3506 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3507
3508 mutex_exit(&ixgbe->gen_lock);
3509 }
3510
3511 /*
3512 * ixgbe_link_timer - timer for link status detection
3513 */
3514 static void
3515 ixgbe_link_timer(void *arg)
3516 {
3517 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3518
3519 mutex_enter(&ixgbe->gen_lock);
3520 ixgbe_driver_link_check(ixgbe);
3521 mutex_exit(&ixgbe->gen_lock);
3522 }
3523
3524 /*
3525 * ixgbe_local_timer - Driver watchdog function.
3526 *
3527 * This function will handle the transmit stall check and other routines.
3528 */
3529 static void
3530 ixgbe_local_timer(void *arg)
3531 {
3655
3656 /*
3657 * Look up the OBP property "local-mac-address?". If the user has set
3658 * 'local-mac-address? = false', use "the system address" instead.
3659 */
3660 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3661 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3662 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3663 if (localetheraddr(NULL, &sysaddr) != 0) {
3664 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3665 found = B_TRUE;
3666 }
3667 }
3668 ddi_prop_free(bytes);
3669 }
3670
3671 /*
3672 * Finally(!), if there's a valid "mac-address" property (created
3673 * if we netbooted from this interface), we must use this instead
3674 * of any of the above to ensure that the NFS/install server doesn't
3675 * get confused by the address changing as Solaris takes over!
3676 */
3677 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3678 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3679 if (err == DDI_PROP_SUCCESS) {
3680 if (nelts == ETHERADDRL) {
3681 while (nelts--)
3682 hw->mac.addr[nelts] = bytes[nelts];
3683 found = B_TRUE;
3684 }
3685 ddi_prop_free(bytes);
3686 }
3687
3688 if (found) {
3689 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3690 return (B_TRUE);
3691 }
3692 #else
3693 _NOTE(ARGUNUSED(ixgbe));
3694 #endif
3695
3845 */
3846 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3847 /* enable autoclear but not on bits 29:20 */
3848 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3849
3850 /* general purpose interrupt enable */
3851 gpie |= (IXGBE_GPIE_MSIX_MODE
3852 | IXGBE_GPIE_PBA_SUPPORT
3853 | IXGBE_GPIE_OCD
3854 | IXGBE_GPIE_EIAME);
3855 /*
3856 * non-msi-x mode
3857 */
3858 } else {
3859
3860 /* disable autoclear, leave gpie at default */
3861 eiac = 0;
3862
3863 /*
3864 * General purpose interrupt enable.
3865 * For 82599 or X540, extended interrupt automask enable
3866 * only in MSI or MSI-X mode
3867 */
3868 if ((hw->mac.type == ixgbe_mac_82598EB) ||
3869 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3870 gpie |= IXGBE_GPIE_EIAME;
3871 }
3872 }
3873
3874 /* Enable specific "other" interrupt types */
3875 switch (hw->mac.type) {
3876 case ixgbe_mac_82598EB:
3877 gpie |= ixgbe->capab->other_gpie;
3878 break;
3879
3880 case ixgbe_mac_82599EB:
3881 case ixgbe_mac_X540:
3882 gpie |= ixgbe->capab->other_gpie;
3883
3884 /* Enable RSC Delay 8us when LRO enabled */
3885 if (ixgbe->lro_enable) {
3886 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3887 }
3888 break;
3889
3890 default:
3891 break;
3892 }
3893
3894 /* write to interrupt control registers */
3895 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3896 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3897 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3898 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3899 IXGBE_WRITE_FLUSH(hw);
3900 }
3901
4056 &atlas);
4057 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4058 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4059 atlas);
4060
4061 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4062 &atlas);
4063 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4064 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4065 atlas);
4066
4067 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4068 &atlas);
4069 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4070 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4071 atlas);
4072 break;
4073
4074 case ixgbe_mac_82599EB:
4075 case ixgbe_mac_X540:
4076 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4077 reg |= (IXGBE_AUTOC_FLU |
4078 IXGBE_AUTOC_10G_KX4);
4079 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4080
4081 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4082 B_FALSE, B_TRUE);
4083 break;
4084
4085 default:
4086 break;
4087 }
4088 }
4089
4090 #pragma inline(ixgbe_intr_rx_work)
4091 /*
4092 * ixgbe_intr_rx_work - RX processing of ISR.
4093 */
4094 static void
4095 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4096 {
4097 mblk_t *mp;
4098
4099 mutex_enter(&rx_ring->rx_lock);
4100
4101 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4102 mutex_exit(&rx_ring->rx_lock);
4122
4123 /*
4124 * Schedule the re-transmit
4125 */
4126 if (tx_ring->reschedule &&
4127 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4128 tx_ring->reschedule = B_FALSE;
4129 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4130 tx_ring->ring_handle);
4131 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4132 }
4133 }
4134
4135 #pragma inline(ixgbe_intr_other_work)
4136 /*
4137 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4138 */
4139 static void
4140 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4141 {
4142 ASSERT(mutex_owned(&ixgbe->gen_lock));
4143
4144 /*
4145 * handle link status change
4146 */
4147 if (eicr & IXGBE_EICR_LSC) {
4148 ixgbe_driver_link_check(ixgbe);
4149 ixgbe_get_hw_state(ixgbe);
4150 }
4151
4152 /*
4153 * check for fan failure on adapters with fans
4154 */
4155 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4156 (eicr & IXGBE_EICR_GPI_SDP1)) {
4157 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4158
4159 /*
4160 * Disable the adapter interrupts
4161 */
4164 /*
4165 * Disable Rx/Tx units
4166 */
4167 (void) ixgbe_stop_adapter(&ixgbe->hw);
4168
4169 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4170 ixgbe_error(ixgbe,
4171 "Problem: Network adapter has been stopped "
4172 "because the fan has stopped.\n");
4173 ixgbe_error(ixgbe,
4174 "Action: Replace the adapter.\n");
4175
4176 /* re-enable the interrupt, which was automasked */
4177 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4178 }
4179
4180 /*
4181 * Do SFP check for adapters with hot-plug capability
4182 */
4183 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4184 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4185 ixgbe->eicr = eicr;
4186 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4187 ixgbe_sfp_check, (void *)ixgbe,
4188 DDI_NOSLEEP)) != DDI_SUCCESS) {
4189 ixgbe_log(ixgbe, "No memory available to dispatch "
4190 "taskq for SFP check");
4191 }
4192 }
4193
4194 /*
4195 * Do over-temperature check for adapters with temp sensor
4196 */
4197 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4198 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4199 ixgbe->eicr = eicr;
4200 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4201 ixgbe_overtemp_check, (void *)ixgbe,
4202 DDI_NOSLEEP)) != DDI_SUCCESS) {
4203 ixgbe_log(ixgbe, "No memory available to dispatch "
4204 "taskq for overtemp check");
4205 }
4206 }
4207 }
4208
4209 /*
4210 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4211 */
4212 static uint_t
4213 ixgbe_intr_legacy(void *arg1, void *arg2)
4214 {
4215 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4216 struct ixgbe_hw *hw = &ixgbe->hw;
4217 ixgbe_tx_ring_t *tx_ring;
4218 ixgbe_rx_ring_t *rx_ring;
4219 uint32_t eicr;
4220 mblk_t *mp;
4221 boolean_t tx_reschedule;
4222 uint_t result;
4223
4224 _NOTE(ARGUNUSED(arg2));
4225
4226 mutex_enter(&ixgbe->gen_lock);
4275 */
4276 tx_ring = &ixgbe->tx_rings[0];
4277 tx_ring->tx_recycle(tx_ring);
4278
4279 /*
4280 * Schedule the re-transmit
4281 */
4282 tx_reschedule = (tx_ring->reschedule &&
4283 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4284 }
4285
4286 /* any interrupt type other than tx/rx */
4287 if (eicr & ixgbe->capab->other_intr) {
4288 switch (hw->mac.type) {
4289 case ixgbe_mac_82598EB:
4290 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4291 break;
4292
4293 case ixgbe_mac_82599EB:
4294 case ixgbe_mac_X540:
4295 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4296 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4297 break;
4298
4299 default:
4300 break;
4301 }
4302 ixgbe_intr_other_work(ixgbe, eicr);
4303 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4304 }
4305
4306 mutex_exit(&ixgbe->gen_lock);
4307
4308 result = DDI_INTR_CLAIMED;
4309 } else {
4310 mutex_exit(&ixgbe->gen_lock);
4311
4312 /*
4313 * No interrupt cause bits set: don't claim this interrupt.
4314 */
4369 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4370 }
4371
4372 /*
4373 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4374 */
4375 if (eicr & 0x2) {
4376 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4377 }
4378
4379 /* any interrupt type other than tx/rx */
4380 if (eicr & ixgbe->capab->other_intr) {
4381 mutex_enter(&ixgbe->gen_lock);
4382 switch (hw->mac.type) {
4383 case ixgbe_mac_82598EB:
4384 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4385 break;
4386
4387 case ixgbe_mac_82599EB:
4388 case ixgbe_mac_X540:
4389 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4390 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4391 break;
4392
4393 default:
4394 break;
4395 }
4396 ixgbe_intr_other_work(ixgbe, eicr);
4397 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4398 mutex_exit(&ixgbe->gen_lock);
4399 }
4400
4401 /* re-enable the interrupts which were automasked */
4402 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4403
4404 return (DDI_INTR_CLAIMED);
4405 }
4406
4407 /*
4408 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4449 DDI_FM_OK) {
4450 ddi_fm_service_impact(ixgbe->dip,
4451 DDI_SERVICE_DEGRADED);
4452 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4453 return (DDI_INTR_CLAIMED);
4454 }
4455
4456 /*
4457 * Check "other" cause bits: any interrupt type other than tx/rx
4458 */
4459 if (eicr & ixgbe->capab->other_intr) {
4460 mutex_enter(&ixgbe->gen_lock);
4461 switch (hw->mac.type) {
4462 case ixgbe_mac_82598EB:
4463 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4464 ixgbe_intr_other_work(ixgbe, eicr);
4465 break;
4466
4467 case ixgbe_mac_82599EB:
4468 case ixgbe_mac_X540:
4469 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4470 ixgbe_intr_other_work(ixgbe, eicr);
4471 break;
4472
4473 default:
4474 break;
4475 }
4476 mutex_exit(&ixgbe->gen_lock);
4477 }
4478
4479 /* re-enable the interrupts which were automasked */
4480 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4481 }
4482
4483 return (DDI_INTR_CLAIMED);
4484 }
4485
4486 /*
4487 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4488 *
4538 ixgbe_log(ixgbe,
4539 "MSI-X not used, force rings and groups number to 1");
4540
4541 /*
4542 * Install MSI interrupts
4543 */
4544 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4545 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4546 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4547 if (rc == IXGBE_SUCCESS)
4548 return (IXGBE_SUCCESS);
4549
4550 ixgbe_log(ixgbe,
4551 "Allocate MSI failed, trying Legacy interrupts...");
4552 }
4553
4554 /*
4555 * Install legacy interrupts
4556 */
4557 if (intr_types & DDI_INTR_TYPE_FIXED) {
4558 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4559 if (rc == IXGBE_SUCCESS)
4560 return (IXGBE_SUCCESS);
4561
4562 ixgbe_log(ixgbe,
4563 "Allocate Legacy interrupts failed");
4564 }
4565
4566 /*
4567 * If none of the 3 types succeeded, return failure
4568 */
4569 return (IXGBE_FAILURE);
4570 }
4571
4572 /*
4573 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4574 *
4575 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4576 * if fewer than 2 handles are available, return failure.
4577 * Upon success, this maps the vectors to rx and tx rings for
4850 int8_t cause)
4851 {
4852 struct ixgbe_hw *hw = &ixgbe->hw;
4853 u32 ivar, index;
4854
4855 switch (hw->mac.type) {
4856 case ixgbe_mac_82598EB:
4857 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4858 if (cause == -1) {
4859 cause = 0;
4860 }
4861 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4862 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4863 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4864 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4865 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4866 break;
4867
4868 case ixgbe_mac_82599EB:
4869 case ixgbe_mac_X540:
4870 if (cause == -1) {
4871 /* other causes */
4872 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4873 index = (intr_alloc_entry & 1) * 8;
4874 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4875 ivar &= ~(0xFF << index);
4876 ivar |= (msix_vector << index);
4877 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4878 } else {
4879 /* tx or rx causes */
4880 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4881 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4882 ivar = IXGBE_READ_REG(hw,
4883 IXGBE_IVAR(intr_alloc_entry >> 1));
4884 ivar &= ~(0xFF << index);
4885 ivar |= (msix_vector << index);
4886 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4887 ivar);
4888 }
4889 break;
4904 static void
4905 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4906 {
4907 struct ixgbe_hw *hw = &ixgbe->hw;
4908 u32 ivar, index;
4909
4910 switch (hw->mac.type) {
4911 case ixgbe_mac_82598EB:
4912 if (cause == -1) {
4913 cause = 0;
4914 }
4915 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4916 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4917 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4918 (intr_alloc_entry & 0x3)));
4919 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4920 break;
4921
4922 case ixgbe_mac_82599EB:
4923 case ixgbe_mac_X540:
4924 if (cause == -1) {
4925 /* other causes */
4926 index = (intr_alloc_entry & 1) * 8;
4927 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4928 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4929 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4930 } else {
4931 /* tx or rx causes */
4932 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4933 ivar = IXGBE_READ_REG(hw,
4934 IXGBE_IVAR(intr_alloc_entry >> 1));
4935 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4936 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4937 ivar);
4938 }
4939 break;
4940
4941 default:
4942 break;
4943 }
4954 static void
4955 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4956 {
4957 struct ixgbe_hw *hw = &ixgbe->hw;
4958 u32 ivar, index;
4959
4960 switch (hw->mac.type) {
4961 case ixgbe_mac_82598EB:
4962 if (cause == -1) {
4963 cause = 0;
4964 }
4965 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4966 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4967 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4968 (intr_alloc_entry & 0x3)));
4969 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4970 break;
4971
4972 case ixgbe_mac_82599EB:
4973 case ixgbe_mac_X540:
4974 if (cause == -1) {
4975 /* other causes */
4976 index = (intr_alloc_entry & 1) * 8;
4977 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4978 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4979 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4980 } else {
4981 /* tx or rx causes */
4982 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4983 ivar = IXGBE_READ_REG(hw,
4984 IXGBE_IVAR(intr_alloc_entry >> 1));
4985 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4986 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4987 ivar);
4988 }
4989 break;
4990
4991 default:
4992 break;
4993 }
4997 * Convert the rx ring index driver maintained to the rx ring index
4998 * in h/w.
4999 */
5000 static uint32_t
5001 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5002 {
5003
5004 struct ixgbe_hw *hw = &ixgbe->hw;
5005 uint32_t rx_ring_per_group, hw_rx_index;
5006
5007 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5008 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5009 return (sw_rx_index);
5010 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5011 switch (hw->mac.type) {
5012 case ixgbe_mac_82598EB:
5013 return (sw_rx_index);
5014
5015 case ixgbe_mac_82599EB:
5016 case ixgbe_mac_X540:
5017 return (sw_rx_index * 2);
5018
5019 default:
5020 break;
5021 }
5022 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5023 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5024
5025 switch (hw->mac.type) {
5026 case ixgbe_mac_82598EB:
5027 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5028 16 + (sw_rx_index % rx_ring_per_group);
5029 return (hw_rx_index);
5030
5031 case ixgbe_mac_82599EB:
5032 case ixgbe_mac_X540:
5033 if (ixgbe->num_rx_groups > 32) {
5034 hw_rx_index = (sw_rx_index /
5035 rx_ring_per_group) * 2 +
5036 (sw_rx_index % rx_ring_per_group);
5037 } else {
5038 hw_rx_index = (sw_rx_index /
5039 rx_ring_per_group) * 4 +
5040 (sw_rx_index % rx_ring_per_group);
5041 }
5042 return (hw_rx_index);
5043
5044 default:
5045 break;
5046 }
5047 }
5048
5049 /*
5050 * Should never reach. Just to make compiler happy.
5051 */
5052 return (sw_rx_index);
5118 static void
5119 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5120 {
5121 struct ixgbe_hw *hw = &ixgbe->hw;
5122 ixgbe_intr_vector_t *vect; /* vector bitmap */
5123 int r_idx; /* ring index */
5124 int v_idx; /* vector index */
5125 uint32_t hw_index;
5126
5127 /*
5128 * Clear any previous entries
5129 */
5130 switch (hw->mac.type) {
5131 case ixgbe_mac_82598EB:
5132 for (v_idx = 0; v_idx < 25; v_idx++)
5133 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5134 break;
5135
5136 case ixgbe_mac_82599EB:
5137 case ixgbe_mac_X540:
5138 for (v_idx = 0; v_idx < 64; v_idx++)
5139 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5140 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5141 break;
5142
5143 default:
5144 break;
5145 }
5146
5147 /*
5148 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5149 * tx rings[0] will use RTxQ[1].
5150 */
5151 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5152 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5153 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5154 return;
5155 }
5156
5157 /*
5293 for (i = 0; i < ixgbe->intr_cnt; i++) {
5294 rc = ddi_intr_disable(ixgbe->htable[i]);
5295 if (rc != DDI_SUCCESS) {
5296 ixgbe_log(ixgbe,
5297 "Disable intr failed: %d", rc);
5298 return (IXGBE_FAILURE);
5299 }
5300 }
5301 }
5302
5303 return (IXGBE_SUCCESS);
5304 }
5305
5306 /*
5307 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5308 */
5309 static void
5310 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5311 {
5312 struct ixgbe_hw *hw = &ixgbe->hw;
5313 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5314 boolean_t link_up = B_FALSE;
5315 uint32_t pcs1g_anlp = 0;
5316 uint32_t pcs1g_ana = 0;
5317 boolean_t autoneg = B_FALSE;
5318
5319 ASSERT(mutex_owned(&ixgbe->gen_lock));
5320 ixgbe->param_lp_1000fdx_cap = 0;
5321 ixgbe->param_lp_100fdx_cap = 0;
5322
5323 /* check for link, don't wait */
5324 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5325 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5326
5327 if (link_up) {
5328 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5329
5330 ixgbe->param_lp_1000fdx_cap =
5331 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5332 ixgbe->param_lp_100fdx_cap =
5333 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5334 }
5335
5336 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5337
5338 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5339 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5340 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5341 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5342 }
5343
5344 /*
5345 * ixgbe_get_driver_control - Notify that driver is in control of device.
5346 */
5347 static void
5348 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5349 {
5350 uint32_t ctrl_ext;
5351
5352 /*
5353 * Notify firmware that driver is in control of device
5354 */
5355 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5356 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5357 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5358 }
5359
5360 /*
5361 * ixgbe_release_driver_control - Notify that driver is no longer in control
|
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
71 static void ixgbe_setup_multicst(ixgbe_t *);
72 static void ixgbe_get_hw_state(ixgbe_t *);
73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
74 static void ixgbe_get_conf(ixgbe_t *);
75 static void ixgbe_init_params(ixgbe_t *);
76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
77 static void ixgbe_driver_link_check(ixgbe_t *);
78 static void ixgbe_sfp_check(void *);
79 static void ixgbe_overtemp_check(void *);
80 static void ixgbe_phy_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 0x200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 IXGBE_EICR_LSC, /* "other" interrupt types handled */
349 0, /* "other" interrupt types enable mask */
350 (IXGBE_FLAG_RSS_CAPABLE
351 | IXGBE_FLAG_VMDQ_CAPABLE
352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
353 };
354
355 /*
356 * Module Initialization Functions.
357 */
358
359 int
360 _init(void)
361 {
362 int status;
363
364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
365
366 status = mod_install(&ixgbe_modlinkage);
367
368 if (status != DDI_SUCCESS) {
369 mac_fini_ops(&ixgbe_dev_ops);
370 }
371
372 return (status);
373 }
374
435 }
436
437 /* Get the device instance */
438 instance = ddi_get_instance(devinfo);
439
440 /* Allocate memory for the instance data structure */
441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
442
443 ixgbe->dip = devinfo;
444 ixgbe->instance = instance;
445
446 hw = &ixgbe->hw;
447 osdep = &ixgbe->osdep;
448 hw->back = osdep;
449 osdep->ixgbe = ixgbe;
450
451 /* Attach the instance pointer to the dev_info data structure */
452 ddi_set_driver_private(devinfo, ixgbe);
453
454 /*
455 * Initialize for FMA support
456 */
457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
460 ixgbe_fm_init(ixgbe);
461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
462
463 /*
464 * Map PCI config space registers
465 */
466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
467 ixgbe_error(ixgbe, "Failed to map PCI configurations");
468 goto attach_fail;
469 }
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
471
472 /*
473 * Identify the chipset family
474 */
475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
546 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
547 ixgbe_error(ixgbe, "sfp_taskq create failed");
548 goto attach_fail;
549 }
550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
551
552 /*
553 * Create a taskq for over-temp
554 */
555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
557 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
558 ixgbe_error(ixgbe, "overtemp_taskq create failed");
559 goto attach_fail;
560 }
561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
562
563 /*
564 * Create a taskq for processing external PHY interrupts
565 */
566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
568 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
569 ixgbe_error(ixgbe, "phy_taskq create failed");
570 goto attach_fail;
571 }
572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
573
574 /*
575 * Initialize driver parameters
576 */
577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize driver settings");
579 goto attach_fail;
580 }
581
582 /*
583 * Initialize mutexes for this device.
584 * Do this before enabling the interrupt handler and
585 * register the softint to avoid the condition where
586 * interrupt handler can try using uninitialized mutex.
587 */
588 ixgbe_init_locks(ixgbe);
589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
590
591 /*
592 * Initialize chipset hardware
593 */
594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
595 ixgbe_error(ixgbe, "Failed to initialize adapter");
596 goto attach_fail;
597 }
598 ixgbe->link_check_complete = B_FALSE;
599 ixgbe->link_check_hrtime = gethrtime() +
600 (IXGBE_LINK_UP_TIME * 100000000ULL);
601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
602
603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
605 goto attach_fail;
606 }
607
608 /*
609 * Initialize adapter capabilities
610 */
611 ixgbe_init_params(ixgbe);
612
613 /*
614 * Initialize statistics
615 */
616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
617 ixgbe_error(ixgbe, "Failed to initialize statistics");
618 goto attach_fail;
619 }
620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
621
622 /*
623 * Register the driver to the MAC
624 */
625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
626 ixgbe_error(ixgbe, "Failed to register MAC");
627 goto attach_fail;
628 }
629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
631
632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
634 if (ixgbe->periodic_id == 0) {
635 ixgbe_error(ixgbe, "Failed to add the link check timer");
636 goto attach_fail;
637 }
638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
639
640 /*
641 * Now that mutex locks are initialized, and the chip is also
642 * initialized, enable interrupts.
643 */
644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
646 goto attach_fail;
647 }
648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
649
650 ixgbe_log(ixgbe, "%s", ixgbe_ident);
651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
652
653 return (DDI_SUCCESS);
654
655 attach_fail:
656 ixgbe_unconfigure(devinfo, ixgbe);
657 return (DDI_FAILURE);
658 }
659
660 /*
661 * ixgbe_detach - Driver detach.
662 *
663 * The detach() function is the complement of the attach routine.
664 * If cmd is set to DDI_DETACH, detach() is used to remove the
665 * state associated with a given instance of a device node
666 * prior to the removal of that instance from the system.
667 *
668 * The detach() function will be called once for each instance
669 * of the device for which there has been a successful attach()
670 * once there are no longer any opens on the device.
813 */
814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
815 ixgbe_rem_intr_handlers(ixgbe);
816 }
817
818 /*
819 * Remove taskq for sfp-status-change
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
822 ddi_taskq_destroy(ixgbe->sfp_taskq);
823 }
824
825 /*
826 * Remove taskq for over-temp
827 */
828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
829 ddi_taskq_destroy(ixgbe->overtemp_taskq);
830 }
831
832 /*
833 * Remove taskq for external PHYs
834 */
835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
836 ddi_taskq_destroy(ixgbe->phy_taskq);
837 }
838
839 /*
840 * Remove interrupts
841 */
842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
843 ixgbe_rem_intrs(ixgbe);
844 }
845
846 /*
847 * Unregister interrupt callback handler
848 */
849 (void) ddi_cb_unregister(ixgbe->cb_hdl);
850
851 /*
852 * Remove driver properties
853 */
854 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
855 (void) ddi_prop_remove_all(devinfo);
856 }
857
858 /*
859 * Stop the chipset
989 case ixgbe_mac_82599EB:
990 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
991 ixgbe->capab = &ixgbe_82599eb_cap;
992
993 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
994 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
995 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
996 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
997 }
998 break;
999
1000 case ixgbe_mac_X540:
1001 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1002 ixgbe->capab = &ixgbe_X540_cap;
1003 /*
1004 * For now, X540 is all set in its capab structure.
1005 * As other X540 variants show up, things can change here.
1006 */
1007 break;
1008
1009 case ixgbe_mac_X550:
1010 case ixgbe_mac_X550EM_x:
1011 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1012 ixgbe->capab = &ixgbe_X550_cap;
1013
1014 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
1015 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1016
1017 /*
1018 * Link detection on X552 SFP+ and X552/X557-AT
1019 */
1020 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1021 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1022 ixgbe->capab->other_intr |=
1023 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1024 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1025 }
1026 break;
1027
1028 default:
1029 IXGBE_DEBUGLOG_1(ixgbe,
1030 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1031 hw->mac.type);
1032 return (IXGBE_FAILURE);
1033 }
1034
1035 return (IXGBE_SUCCESS);
1036 }
1037
1038 /*
1039 * ixgbe_regs_map - Map the device registers.
1040 *
1041 */
1042 static int
1043 ixgbe_regs_map(ixgbe_t *ixgbe)
1044 {
1045 dev_info_t *devinfo = ixgbe->dip;
1046 struct ixgbe_hw *hw = &ixgbe->hw;
1047 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1062 (caddr_t *)&hw->hw_addr, 0,
1063 mem_size, &ixgbe_regs_acc_attr,
1064 &osdep->reg_handle)) != DDI_SUCCESS) {
1065 return (IXGBE_FAILURE);
1066 }
1067
1068 return (IXGBE_SUCCESS);
1069 }
1070
1071 /*
1072 * ixgbe_init_properties - Initialize driver properties.
1073 */
1074 static void
1075 ixgbe_init_properties(ixgbe_t *ixgbe)
1076 {
1077 /*
1078 * Get conf file properties, including link settings
1079 * jumbo frames, ring number, descriptor number, etc.
1080 */
1081 ixgbe_get_conf(ixgbe);
1082 }
1083
1084 /*
1085 * ixgbe_init_driver_settings - Initialize driver settings.
1086 *
1087 * The settings include hardware function pointers, bus information,
1088 * rx/tx rings settings, link state, and any other parameters that
1089 * need to be setup during driver initialization.
1090 */
1091 static int
1092 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1093 {
1094 struct ixgbe_hw *hw = &ixgbe->hw;
1095 dev_info_t *devinfo = ixgbe->dip;
1096 ixgbe_rx_ring_t *rx_ring;
1097 ixgbe_rx_group_t *rx_group;
1098 ixgbe_tx_ring_t *tx_ring;
1099 uint32_t rx_size;
1100 uint32_t tx_size;
1101 uint32_t ring_per_group;
1296 ixgbe_stop(ixgbe, B_FALSE);
1297
1298 mutex_exit(&ixgbe->gen_lock);
1299
1300 /*
1301 * Disable and stop the watchdog timer
1302 */
1303 ixgbe_disable_watchdog_timer(ixgbe);
1304
1305 return (DDI_SUCCESS);
1306 }
1307
1308 /*
1309 * ixgbe_init - Initialize the device.
1310 */
1311 static int
1312 ixgbe_init(ixgbe_t *ixgbe)
1313 {
1314 struct ixgbe_hw *hw = &ixgbe->hw;
1315 u8 pbanum[IXGBE_PBANUM_LENGTH];
1316 int rv;
1317
1318 mutex_enter(&ixgbe->gen_lock);
1319
1320 /*
1321 * Configure/Initialize hardware
1322 */
1323 rv = ixgbe_init_hw(hw);
1324 if (rv != IXGBE_SUCCESS) {
1325 switch (rv) {
1326
1327 /*
1328 * The first three errors are not prohibitive to us progressing
1329 * further, and are maily advisory in nature. In the case of a
1330 * SFP module not being present or not deemed supported by the
1331 * common code, we adivse the operator of this fact but carry on
1332 * instead of failing hard, as SFPs can be inserted or replaced
1333 * while the driver is running. In the case of a unknown error,
1334 * we fail-hard, logging the reason and emitting a FMA event.
1335 */
1336 case IXGBE_ERR_EEPROM_VERSION:
1337 ixgbe_error(ixgbe,
1338 "This Intel 10Gb Ethernet device is pre-release and"
1339 " contains outdated firmware. Please contact your"
1340 " hardware vendor for a replacement.");
1341 break;
1342 case IXGBE_ERR_SFP_NOT_PRESENT:
1343 ixgbe_error(ixgbe,
1344 "No SFP+ module detected on this interface. Please "
1345 "install a supported SFP+ module for this "
1346 "interface to become operational.");
1347 break;
1348 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1349 ixgbe_error(ixgbe,
1350 "Unsupported SFP+ module detected. Please replace "
1351 "it with a supported SFP+ module per Intel "
1352 "documentation, or bypass this check with "
1353 "allow_unsupported_sfp=1 in ixgbe.conf.");
1354 break;
1355 default:
1356 ixgbe_error(ixgbe,
1357 "Failed to initialize hardware. ixgbe_init_hw "
1358 "returned %d", rv);
1359 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1360 goto init_fail;
1361 }
1362 }
1363
1364 /*
1365 * Need to init eeprom before validating the checksum.
1366 */
1367 if (ixgbe_init_eeprom_params(hw) < 0) {
1368 ixgbe_error(ixgbe,
1369 "Unable to intitialize the eeprom interface.");
1370 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1371 goto init_fail;
1372 }
1373
1374 /*
1375 * NVM validation
1376 */
1377 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1378 /*
1379 * Some PCI-E parts fail the first check due to
1380 * the link being in sleep state. Call it again,
1381 * if it fails a second time it's a real issue.
1382 */
1383 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1384 ixgbe_error(ixgbe,
1385 "Invalid NVM checksum. Please contact "
1386 "the vendor to update the NVM.");
1387 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1388 goto init_fail;
1389 }
1390 }
1391
1392 /*
1393 * Setup default flow control thresholds - enable/disable
1394 * & flow control type is controlled by ixgbe.conf
1395 */
1396 hw->fc.high_water[0] = DEFAULT_FCRTH;
1397 hw->fc.low_water[0] = DEFAULT_FCRTL;
1398 hw->fc.pause_time = DEFAULT_FCPAUSE;
1399 hw->fc.send_xon = B_TRUE;
1400
1401 /*
1402 * Initialize flow control
1403 */
1404 (void) ixgbe_start_hw(hw);
1405
1406 /*
1407 * Initialize link settings
1408 */
1409 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1410
1411 /*
1412 * Initialize the chipset hardware
1413 */
1414 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1415 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1416 goto init_fail;
1417 }
1418
1419 /*
1420 * Read identifying information and place in devinfo.
1421 */
1422 pbanum[0] = '\0';
1423 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1424 if (*pbanum != '\0') {
1425 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1426 "printed-board-assembly", (char *)pbanum);
1434 return (IXGBE_SUCCESS);
1435
1436 init_fail:
1437 /*
1438 * Reset PHY
1439 */
1440 (void) ixgbe_reset_phy(hw);
1441
1442 mutex_exit(&ixgbe->gen_lock);
1443 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1444 return (IXGBE_FAILURE);
1445 }
1446
1447 /*
1448 * ixgbe_chip_start - Initialize and start the chipset hardware.
1449 */
1450 static int
1451 ixgbe_chip_start(ixgbe_t *ixgbe)
1452 {
1453 struct ixgbe_hw *hw = &ixgbe->hw;
1454 int i;
1455
1456 ASSERT(mutex_owned(&ixgbe->gen_lock));
1457
1458 /*
1459 * Get the mac address
1460 * This function should handle SPARC case correctly.
1461 */
1462 if (!ixgbe_find_mac_address(ixgbe)) {
1463 ixgbe_error(ixgbe, "Failed to get the mac address");
1464 return (IXGBE_FAILURE);
1465 }
1466
1467 /*
1468 * Validate the mac address
1469 */
1470 (void) ixgbe_init_rx_addrs(hw);
1471 if (!is_valid_mac_addr(hw->mac.addr)) {
1472 ixgbe_error(ixgbe, "Invalid mac address");
1473 return (IXGBE_FAILURE);
1474 }
1475
1476 /*
1477 * Re-enable relaxed ordering for performance. It is disabled
1478 * by default in the hardware init.
1479 */
1480 if (ixgbe->relax_order_enable == B_TRUE)
1481 ixgbe_enable_relaxed_ordering(hw);
1482
1483 /*
1484 * Setup adapter interrupt vectors
1485 */
1486 ixgbe_setup_adapter_vector(ixgbe);
1487
1488 /*
1489 * Initialize unicast addresses.
1490 */
1491 ixgbe_init_unicst(ixgbe);
1492
1493 /*
1494 * Setup and initialize the mctable structures.
1495 */
1496 ixgbe_setup_multicst(ixgbe);
1497
1498 /*
1499 * Set interrupt throttling rate
1500 */
1501 for (i = 0; i < ixgbe->intr_cnt; i++) {
1502 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1503 }
1504
1505 /*
1506 * Disable Wake-on-LAN
1507 */
1508 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1509
1510 /*
1511 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1512 * Due to issues with EEE in e1000g/igb, we disable this by default
1513 * as a precautionary measure.
1514 *
1515 * Currently, the only known adapter which supports EEE in the ixgbe
1516 * line is 8086,15AB (IXGBE_DEV_ID_X550EM_X_KR), and only after the
1517 * first revision of it, as well as any X550 with MAC type 6 (non-EM)
1518 */
1519 (void) ixgbe_setup_eee(hw, B_FALSE);
1520
1521 /*
1522 * Turn on any present SFP Tx laser
1523 */
1524 ixgbe_enable_tx_laser(hw);
1525
1526 /*
1527 * Power on the PHY
1528 */
1529 (void) ixgbe_set_phy_power(hw, B_TRUE);
1530
1531 /*
1532 * Save the state of the PHY
1533 */
1534 ixgbe_get_hw_state(ixgbe);
1535
1536 /*
1537 * Make sure driver has control
1538 */
1539 ixgbe_get_driver_control(hw);
1540
1541 return (IXGBE_SUCCESS);
1542 }
1543
1544 /*
1545 * ixgbe_chip_stop - Stop the chipset hardware
1546 */
1547 static void
1548 ixgbe_chip_stop(ixgbe_t *ixgbe)
1549 {
1550 struct ixgbe_hw *hw = &ixgbe->hw;
1551 int rv;
1552
1553 ASSERT(mutex_owned(&ixgbe->gen_lock));
1554
1555 /*
1556 * Stop interupt generation and disable Tx unit
1557 */
1558 hw->adapter_stopped = B_FALSE;
1559 (void) ixgbe_stop_adapter(hw);
1560
1561 /*
1562 * Reset the chipset
1563 */
1564 (void) ixgbe_reset_hw(hw);
1565
1566 /*
1567 * Reset PHY
1568 */
1569 (void) ixgbe_reset_phy(hw);
1570
1571 /*
1572 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1573 * the PHY while doing so. Else, just power down the PHY.
1574 */
1575 if (hw->phy.ops.enter_lplu != NULL) {
1576 hw->phy.reset_disable = B_TRUE;
1577 rv = hw->phy.ops.enter_lplu(hw);
1578 if (rv != IXGBE_SUCCESS)
1579 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1580 hw->phy.reset_disable = B_FALSE;
1581 } else {
1582 (void) ixgbe_set_phy_power(hw, B_FALSE);
1583 }
1584
1585 /*
1586 * Turn off any present SFP Tx laser
1587 * Expected for health and safety reasons
1588 */
1589 ixgbe_disable_tx_laser(hw);
1590
1591 /*
1592 * Tell firmware driver is no longer in control
1593 */
1594 ixgbe_release_driver_control(hw);
1595
1596 }
1597
1598 /*
1599 * ixgbe_reset - Reset the chipset and re-start the driver.
1600 *
1601 * It involves stopping and re-starting the chipset,
1602 * and re-configuring the rx/tx rings.
1603 */
1604 static int
1605 ixgbe_reset(ixgbe_t *ixgbe)
1606 {
1607 int i;
1608
1609 /*
1610 * Disable and stop the watchdog timer
1611 */
1612 ixgbe_disable_watchdog_timer(ixgbe);
1613
1614 mutex_enter(&ixgbe->gen_lock);
1615
1777 * Otherwise return B_FALSE;
1778 */
1779 for (i = 0; i < RX_DRAIN_TIME; i++) {
1780 done = (ixgbe->rcb_pending == 0);
1781
1782 if (done)
1783 break;
1784
1785 msec_delay(1);
1786 }
1787
1788 return (done);
1789 }
1790
1791 /*
1792 * ixgbe_start - Start the driver/chipset.
1793 */
1794 int
1795 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1796 {
1797 struct ixgbe_hw *hw = &ixgbe->hw;
1798 int i;
1799
1800 ASSERT(mutex_owned(&ixgbe->gen_lock));
1801
1802 if (alloc_buffer) {
1803 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1804 ixgbe_error(ixgbe,
1805 "Failed to allocate software receive rings");
1806 return (IXGBE_FAILURE);
1807 }
1808
1809 /* Allocate buffers for all the rx/tx rings */
1810 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1811 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1812 return (IXGBE_FAILURE);
1813 }
1814
1815 ixgbe->tx_ring_init = B_TRUE;
1816 } else {
1817 ixgbe->tx_ring_init = B_FALSE;
1818 }
1819
1820 for (i = 0; i < ixgbe->num_rx_rings; i++)
1821 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1822 for (i = 0; i < ixgbe->num_tx_rings; i++)
1823 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1824
1825 /*
1826 * Start the chipset hardware
1827 */
1828 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1829 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1830 goto start_failure;
1831 }
1832
1833 /*
1834 * Configure link now for X550
1835 *
1836 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1837 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1838 * the resting state of the link would be the maximum speed that
1839 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1840 * so we never bothered with explicitly setting the link to 10Gb as it
1841 * would already be at that state on driver attach. With X550, we must
1842 * trigger a re-negotiation of the link in order to switch from a LPLU
1843 * 1Gb link to 10Gb (cable and link partner permitting.)
1844 */
1845 if (hw->mac.type == ixgbe_mac_X550 ||
1846 hw->mac.type == ixgbe_mac_X550EM_x) {
1847 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1848 ixgbe_get_hw_state(ixgbe);
1849 }
1850
1851 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1852 goto start_failure;
1853 }
1854
1855 /*
1856 * Setup the rx/tx rings
1857 */
1858 ixgbe_setup_rings(ixgbe);
1859
1860 /*
1861 * ixgbe_start() will be called when resetting, however if reset
1862 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1863 * before enabling the interrupts.
1864 */
1865 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1866 | IXGBE_STALL| IXGBE_OVERTEMP));
1867
1868 /*
1869 * Enable adapter interrupts
1870 * The interrupts must be enabled after the driver state is START
2338
2339 /*
2340 * Setup head & tail pointers
2341 */
2342 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2343 rx_data->ring_size - 1);
2344 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2345
2346 rx_data->rbd_next = 0;
2347 rx_data->lro_first = 0;
2348
2349 /*
2350 * Setup the Receive Descriptor Control Register (RXDCTL)
2351 * PTHRESH=32 descriptors (half the internal cache)
2352 * HTHRESH=0 descriptors (to minimize latency on fetch)
2353 * WTHRESH defaults to 1 (writeback each descriptor)
2354 */
2355 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2356 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2357
2358 /* Not a valid value for 82599, X540 or X550 */
2359 if (hw->mac.type == ixgbe_mac_82598EB) {
2360 reg_val |= 0x0020; /* pthresh */
2361 }
2362 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2363
2364 if (hw->mac.type == ixgbe_mac_82599EB ||
2365 hw->mac.type == ixgbe_mac_X540 ||
2366 hw->mac.type == ixgbe_mac_X550 ||
2367 hw->mac.type == ixgbe_mac_X550EM_x) {
2368 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2369 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2370 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2371 }
2372
2373 /*
2374 * Setup the Split and Replication Receive Control Register.
2375 * Set the rx buffer size and the advanced descriptor type.
2376 */
2377 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2378 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2379 reg_val |= IXGBE_SRRCTL_DROP_EN;
2380 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2381 }
2382
2383 static void
2384 ixgbe_setup_rx(ixgbe_t *ixgbe)
2385 {
2386 ixgbe_rx_ring_t *rx_ring;
2387 struct ixgbe_hw *hw = &ixgbe->hw;
2388 uint32_t reg_val;
2389 uint32_t ring_mapping;
2390 uint32_t i, index;
2391 uint32_t psrtype_rss_bit;
2392
2393 /*
2394 * Ensure that Rx is disabled while setting up
2395 * the Rx unit and Rx descriptor ring(s)
2396 */
2397 ixgbe_disable_rx(hw);
2398
2399 /* PSRTYPE must be configured for 82599 */
2400 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2401 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2402 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2403 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2404 reg_val |= IXGBE_PSRTYPE_L2HDR;
2405 reg_val |= 0x80000000;
2406 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2407 } else {
2408 if (ixgbe->num_rx_groups > 32) {
2409 psrtype_rss_bit = 0x20000000;
2410 } else {
2411 psrtype_rss_bit = 0x40000000;
2412 }
2413 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2414 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2415 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2416 reg_val |= IXGBE_PSRTYPE_L2HDR;
2417 reg_val |= psrtype_rss_bit;
2418 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2419 }
2420 }
2421
2422 /*
2423 * Set filter control in FCTRL to determine types of packets are passed
2424 * up to the driver.
2425 * - Pass broadcast packets.
2426 * - Do not pass flow control pause frames (82598-specific)
2427 */
2428 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2429 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2430 if (hw->mac.type == ixgbe_mac_82598EB) {
2431 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2432 }
2433 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2434
2435 /*
2436 * Hardware checksum settings
2437 */
2438 if (ixgbe->rx_hcksum_enable) {
2439 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2440 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2441 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2442 }
2443
2444 /*
2445 * Setup VMDq and RSS for multiple receive queues
2446 */
2447 switch (ixgbe->classify_mode) {
2448 case IXGBE_CLASSIFY_RSS:
2449 /*
2450 * One group, only RSS is needed when more than
2451 * one ring enabled.
2452 */
2453 ixgbe_setup_rss(ixgbe);
2454 break;
2455
2456 case IXGBE_CLASSIFY_VMDQ:
2457 /*
2458 * Multiple groups, each group has one ring,
2459 * only VMDq is needed.
2460 */
2461 ixgbe_setup_vmdq(ixgbe);
2462 break;
2463
2464 case IXGBE_CLASSIFY_VMDQ_RSS:
2465 /*
2466 * Multiple groups and multiple rings, both
2467 * VMDq and RSS are needed.
2468 */
2469 ixgbe_setup_vmdq_rss(ixgbe);
2470 break;
2471
2472 default:
2473 break;
2474 }
2475
2476 /*
2477 * Enable the receive unit. This must be done after filter
2478 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2479 * 82598 is the only adapter which defines this RXCTRL option.
2480 */
2481 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2482 if (hw->mac.type == ixgbe_mac_82598EB)
2483 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2484 reg_val |= IXGBE_RXCTRL_RXEN;
2485 (void) ixgbe_enable_rx_dma(hw, reg_val);
2486
2487 /*
2488 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2489 */
2490 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2491 rx_ring = &ixgbe->rx_rings[i];
2492 ixgbe_setup_rx_ring(rx_ring);
2493 }
2494
2495 /*
2496 * Setup the per-ring statistics mapping.
2497 */
2498 ring_mapping = 0;
2499 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2500 index = ixgbe->rx_rings[i].hw_index;
2501 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2502 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2503 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2504 }
2505
2506 /*
2507 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2508 * by four bytes if the packet has a VLAN field, so includes MTU,
2509 * ethernet header and frame check sequence.
2510 * Register is MAXFRS in 82599.
2511 */
2512 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2513 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2514 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2515 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2516 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2517
2518 /*
2519 * Setup Jumbo Frame enable bit
2520 */
2521 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2522 if (ixgbe->default_mtu > ETHERMTU)
2523 reg_val |= IXGBE_HLREG0_JUMBOEN;
2524 else
2525 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2526 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2527
2528 /*
2529 * Setup RSC for multiple receive queues.
2530 */
2531 if (ixgbe->lro_enable) {
2532 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2533 /*
2534 * Make sure rx_buf_size * MAXDESC not greater
2535 * than 65535.
2536 * Intel recommends 4 for MAXDESC field value.
2537 */
2538 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2539 reg_val |= IXGBE_RSCCTL_RSCEN;
2540 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2541 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2542 else
2543 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2544 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2545 }
2546
2661 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2662 tx_ring = &ixgbe->tx_rings[i];
2663 ixgbe_setup_tx_ring(tx_ring);
2664 }
2665
2666 /*
2667 * Setup the per-ring statistics mapping.
2668 */
2669 ring_mapping = 0;
2670 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2671 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2672 if ((i & 0x3) == 0x3) {
2673 switch (hw->mac.type) {
2674 case ixgbe_mac_82598EB:
2675 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2676 ring_mapping);
2677 break;
2678
2679 case ixgbe_mac_82599EB:
2680 case ixgbe_mac_X540:
2681 case ixgbe_mac_X550:
2682 case ixgbe_mac_X550EM_x:
2683 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2684 ring_mapping);
2685 break;
2686
2687 default:
2688 break;
2689 }
2690
2691 ring_mapping = 0;
2692 }
2693 }
2694 if (i & 0x3) {
2695 switch (hw->mac.type) {
2696 case ixgbe_mac_82598EB:
2697 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2698 break;
2699
2700 case ixgbe_mac_82599EB:
2701 case ixgbe_mac_X540:
2702 case ixgbe_mac_X550:
2703 case ixgbe_mac_X550EM_x:
2704 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2705 break;
2706
2707 default:
2708 break;
2709 }
2710 }
2711
2712 /*
2713 * Enable CRC appending and TX padding (for short tx frames)
2714 */
2715 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2716 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2717 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2718
2719 /*
2720 * enable DMA for 82599, X540 and X550 parts
2721 */
2722 if (hw->mac.type == ixgbe_mac_82599EB ||
2723 hw->mac.type == ixgbe_mac_X540 ||
2724 hw->mac.type == ixgbe_mac_X550 ||
2725 hw->mac.type == ixgbe_mac_X550EM_x) {
2726 /* DMATXCTL.TE must be set after all Tx config is complete */
2727 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2728 reg_val |= IXGBE_DMATXCTL_TE;
2729 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2730
2731 /* Disable arbiter to set MTQC */
2732 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2733 reg_val |= IXGBE_RTTDCS_ARBDIS;
2734 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2735 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2736 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2737 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2738 }
2739
2740 /*
2741 * Enabling tx queues ..
2742 * For 82599 must be done after DMATXCTL.TE is set
2743 */
2744 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2745 tx_ring = &ixgbe->tx_rings[i];
2746 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2747 reg_val |= IXGBE_TXDCTL_ENABLE;
2748 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2749 }
2750 }
2751
2752 /*
2753 * ixgbe_setup_rss - Setup receive-side scaling feature.
2754 */
2755 static void
2756 ixgbe_setup_rss(ixgbe_t *ixgbe)
2757 {
2758 struct ixgbe_hw *hw = &ixgbe->hw;
2759 uint32_t mrqc;
2760
2761 /*
2762 * Initialize RETA/ERETA table
2763 */
2764 ixgbe_setup_rss_table(ixgbe);
2765
2766 /*
2767 * Enable RSS & perform hash on these packet types
2768 */
2769 mrqc = IXGBE_MRQC_RSSEN |
2770 IXGBE_MRQC_RSS_FIELD_IPV4 |
2771 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2772 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2773 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2774 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2775 IXGBE_MRQC_RSS_FIELD_IPV6 |
2776 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2777 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2778 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2779 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2780 }
2781
2782 /*
2783 * ixgbe_setup_vmdq - Setup MAC classification feature
2784 */
2785 static void
2786 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2787 {
2788 struct ixgbe_hw *hw = &ixgbe->hw;
2789 uint32_t vmdctl, i, vtctl;
2790
2791 /*
2792 * Setup the VMDq Control register, enable VMDq based on
2793 * packet destination MAC address:
2794 */
2795 switch (hw->mac.type) {
2796 case ixgbe_mac_82598EB:
2797 /*
2798 * VMDq Enable = 1;
2799 * VMDq Filter = 0; MAC filtering
2800 * Default VMDq output index = 0;
2801 */
2802 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2803 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2804 break;
2805
2806 case ixgbe_mac_82599EB:
2807 case ixgbe_mac_X540:
2808 case ixgbe_mac_X550:
2809 case ixgbe_mac_X550EM_x:
2810 /*
2811 * Enable VMDq-only.
2812 */
2813 vmdctl = IXGBE_MRQC_VMDQEN;
2814 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2815
2816 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2817 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2818 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2819 }
2820
2821 /*
2822 * Enable Virtualization and Replication.
2823 */
2824 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2825 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2826
2827 /*
2828 * Enable receiving packets to all VFs
2829 */
2830 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2831 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2832 break;
2833
2834 default:
2835 break;
2836 }
2837 }
2838
2839 /*
2840 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2841 */
2842 static void
2843 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2844 {
2845 struct ixgbe_hw *hw = &ixgbe->hw;
2846 uint32_t i, mrqc;
2847 uint32_t vtctl, vmdctl;
2848
2849 /*
2850 * Initialize RETA/ERETA table
2851 */
2852 ixgbe_setup_rss_table(ixgbe);
2853
2854 /*
2855 * Enable and setup RSS and VMDq
2856 */
2857 switch (hw->mac.type) {
2858 case ixgbe_mac_82598EB:
2859 /*
2860 * Enable RSS & Setup RSS Hash functions
2861 */
2862 mrqc = IXGBE_MRQC_RSSEN |
2863 IXGBE_MRQC_RSS_FIELD_IPV4 |
2864 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2865 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2866 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2867 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2868 IXGBE_MRQC_RSS_FIELD_IPV6 |
2869 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2870 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2871 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2872 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2873
2874 /*
2875 * Enable and Setup VMDq
2876 * VMDq Filter = 0; MAC filtering
2877 * Default VMDq output index = 0;
2878 */
2879 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2880 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2881 break;
2882
2883 case ixgbe_mac_82599EB:
2884 case ixgbe_mac_X540:
2885 case ixgbe_mac_X550:
2886 case ixgbe_mac_X550EM_x:
2887 /*
2888 * Enable RSS & Setup RSS Hash functions
2889 */
2890 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2891 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2892 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2893 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2894 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2895 IXGBE_MRQC_RSS_FIELD_IPV6 |
2896 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2897 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2898 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2899
2900 /*
2901 * Enable VMDq+RSS.
2902 */
2903 if (ixgbe->num_rx_groups > 32) {
2904 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2905 } else {
2906 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2907 }
2908
2909 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2910
2911 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2912 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2913 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2914 }
2915 break;
2916
2917 default:
2918 break;
2919
2920 }
2921
2922 if (hw->mac.type == ixgbe_mac_82599EB ||
2923 hw->mac.type == ixgbe_mac_X540 ||
2924 hw->mac.type == ixgbe_mac_X550 ||
2925 hw->mac.type == ixgbe_mac_X550EM_x) {
2926 /*
2927 * Enable Virtualization and Replication.
2928 */
2929 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2930 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2931
2932 /*
2933 * Enable receiving packets to all VFs
2934 */
2935 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2936 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2937 }
2938 }
2939
2940 /*
2941 * ixgbe_setup_rss_table - Setup RSS table
2942 */
2943 static void
2944 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
2945 {
2946 struct ixgbe_hw *hw = &ixgbe->hw;
2947 uint32_t i, j;
2948 uint32_t random;
2949 uint32_t reta;
2950 uint32_t ring_per_group;
2951 uint32_t ring;
2952 uint32_t table_size;
2953 uint32_t index_mult;
2954 uint32_t rxcsum;
2955
2956 /*
2957 * Set multiplier for RETA setup and table size based on MAC type.
2958 * RETA table sizes vary by model:
2959 *
2960 * 82598, 82599, X540: 128 table entries.
2961 * X550: 512 table entries.
2962 */
2963 index_mult = 0x1;
2964 table_size = 128;
2965 switch (ixgbe->hw.mac.type) {
2966 case ixgbe_mac_82598EB:
2967 index_mult = 0x11;
2968 break;
2969 case ixgbe_mac_X550:
2970 case ixgbe_mac_X550EM_x:
2971 table_size = 512;
2972 break;
2973 default:
2974 break;
2975 }
2976
2977 /*
2978 * Fill out RSS redirection table. The configuation of the indices is
2979 * hardware-dependent.
2980 *
2981 * 82598: 8 bits wide containing two 4 bit RSS indices
2982 * 82599, X540: 8 bits wide containing one 4 bit RSS index
2983 * X550: 8 bits wide containing one 6 bit RSS index
2984 */
2985 reta = 0;
2986 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2987
2988 for (i = 0, j = 0; i < table_size; i++, j++) {
2989 if (j == ring_per_group) j = 0;
2990
2991 /*
2992 * The low 8 bits are for hash value (n+0);
2993 * The next 8 bits are for hash value (n+1), etc.
2994 */
2995 ring = (j * index_mult);
2996 reta = reta >> 8;
2997 reta = reta | (((uint32_t)ring) << 24);
2998
2999 if ((i & 3) == 3)
3000 /*
3001 * The first 128 table entries are programmed into the
3002 * RETA register, with any beyond that (eg; on X550)
3003 * into ERETA.
3004 */
3005 if (i < 128)
3006 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3007 else
3008 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3009 reta);
3010 reta = 0;
3011 }
3012
3013 /*
3014 * Fill out hash function seeds with a random constant
3015 */
3016 for (i = 0; i < 10; i++) {
3017 (void) random_get_pseudo_bytes((uint8_t *)&random,
3018 sizeof (uint32_t));
3019 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3020 }
3021
3022 /*
3023 * Disable Packet Checksum to enable RSS for multiple receive queues.
3024 * It is an adapter hardware limitation that Packet Checksum is
3025 * mutually exclusive with RSS.
3026 */
3027 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3028 rxcsum |= IXGBE_RXCSUM_PCSD;
3029 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3030 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3031 }
3032
3033 /*
3034 * ixgbe_init_unicst - Initialize the unicast addresses.
3035 */
3036 static void
3037 ixgbe_init_unicst(ixgbe_t *ixgbe)
3038 {
3039 struct ixgbe_hw *hw = &ixgbe->hw;
3040 uint8_t *mac_addr;
3041 int slot;
3042 /*
3043 * Here we should consider two situations:
3044 *
3045 * 1. Chipset is initialized at the first time,
3046 * Clear all the multiple unicast addresses.
3047 *
3048 * 2. Chipset is reset
3049 * Recover the multiple unicast addresses from the
3050 * software data structure to the RAR registers.
3051 */
3052 if (!ixgbe->unicst_init) {
3053 /*
3210 /*
3211 * 82598 supports the following combination:
3212 * vmdq no. x rss no.
3213 * [5..16] x 1
3214 * [1..4] x [1..16]
3215 * However 8 rss queue per pool (vmdq) is sufficient for
3216 * most cases.
3217 */
3218 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3219 if (ixgbe->num_rx_groups > 4) {
3220 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3221 } else {
3222 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3223 min(8, ring_per_group);
3224 }
3225
3226 break;
3227
3228 case ixgbe_mac_82599EB:
3229 case ixgbe_mac_X540:
3230 case ixgbe_mac_X550:
3231 case ixgbe_mac_X550EM_x:
3232 /*
3233 * 82599 supports the following combination:
3234 * vmdq no. x rss no.
3235 * [33..64] x [1..2]
3236 * [2..32] x [1..4]
3237 * 1 x [1..16]
3238 * However 8 rss queue per pool (vmdq) is sufficient for
3239 * most cases.
3240 *
3241 * For now, treat X540 and X550 like the 82599.
3242 */
3243 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3244 if (ixgbe->num_rx_groups == 1) {
3245 ixgbe->num_rx_rings = min(8, ring_per_group);
3246 } else if (ixgbe->num_rx_groups <= 32) {
3247 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3248 min(4, ring_per_group);
3249 } else if (ixgbe->num_rx_groups <= 64) {
3250 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3251 min(2, ring_per_group);
3252 }
3253 break;
3254
3255 default:
3256 break;
3257 }
3258
3259 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3260
3261 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3384 * 1 = force interrupt type MSI-X
3385 * 2 = force interrupt type MSI
3386 * 3 = force interrupt type Legacy
3387 */
3388 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3389 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3390
3391 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3392 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3393 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3394 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3395 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3396 0, 1, DEFAULT_LSO_ENABLE);
3397 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3398 0, 1, DEFAULT_LRO_ENABLE);
3399 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3400 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3401 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3402 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3403
3404 /* Head Write Back not recommended for 82599, X540 and X550 */
3405 if (hw->mac.type == ixgbe_mac_82599EB ||
3406 hw->mac.type == ixgbe_mac_X540 ||
3407 hw->mac.type == ixgbe_mac_X550 ||
3408 hw->mac.type == ixgbe_mac_X550EM_x) {
3409 ixgbe->tx_head_wb_enable = B_FALSE;
3410 }
3411
3412 /*
3413 * ixgbe LSO needs the tx h/w checksum support.
3414 * LSO will be disabled if tx h/w checksum is not
3415 * enabled.
3416 */
3417 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3418 ixgbe->lso_enable = B_FALSE;
3419 }
3420
3421 /*
3422 * ixgbe LRO needs the rx h/w checksum support.
3423 * LRO will be disabled if rx h/w checksum is not
3424 * enabled.
3425 */
3426 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3427 ixgbe->lro_enable = B_FALSE;
3428 }
3429
3430 /*
3431 * ixgbe LRO only supported by 82599, X540 and X550
3432 */
3433 if (hw->mac.type == ixgbe_mac_82598EB) {
3434 ixgbe->lro_enable = B_FALSE;
3435 }
3436 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3437 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3438 DEFAULT_TX_COPY_THRESHOLD);
3439 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3440 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3441 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3442 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3443 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3444 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3445 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3446 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3447 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3448
3449 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3450 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3451 DEFAULT_RX_COPY_THRESHOLD);
3452 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3453 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3454 DEFAULT_RX_LIMIT_PER_INTR);
3455
3456 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3457 ixgbe->capab->min_intr_throttle,
3458 ixgbe->capab->max_intr_throttle,
3459 ixgbe->capab->def_intr_throttle);
3460 /*
3461 * 82599, X540 and X550 require the interrupt throttling rate is
3462 * a multiple of 8. This is enforced by the register definiton.
3463 */
3464 if (hw->mac.type == ixgbe_mac_82599EB ||
3465 hw->mac.type == ixgbe_mac_X540 ||
3466 hw->mac.type == ixgbe_mac_X550 ||
3467 hw->mac.type == ixgbe_mac_X550EM_x)
3468 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3469
3470 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3471 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3472 }
3473
3474 static void
3475 ixgbe_init_params(ixgbe_t *ixgbe)
3476 {
3477 struct ixgbe_hw *hw = &ixgbe->hw;
3478 ixgbe_link_speed speeds_supported = 0;
3479 boolean_t negotiate;
3480
3481 /*
3482 * Get a list of speeds the adapter supports. If the hw struct hasn't
3483 * been populated with this information yet, retrieve it from the
3484 * adapter and save it to our own variable.
3485 *
3486 * On certain adapters, such as ones which use SFPs, the contents of
3487 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3488 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3489 * in order to ascertain the speeds which we are capable of supporting,
3490 * and in the case of SFP-equipped adapters, which speed we are
3491 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3492 * we'll go with a default list of speeds as a last resort.
3493 */
3494 speeds_supported = hw->phy.speeds_supported;
3495
3496 if (speeds_supported == 0) {
3497 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3498 &negotiate) != IXGBE_SUCCESS) {
3499 if (hw->mac.type == ixgbe_mac_82598EB) {
3500 speeds_supported =
3501 IXGBE_LINK_SPEED_82598_AUTONEG;
3502 } else {
3503 speeds_supported =
3504 IXGBE_LINK_SPEED_82599_AUTONEG;
3505 }
3506 }
3507 }
3508 ixgbe->speeds_supported = speeds_supported;
3509
3510 /*
3511 * By default, all supported speeds are enabled and advertised.
3512 */
3513 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3514 ixgbe->param_en_10000fdx_cap = 1;
3515 ixgbe->param_adv_10000fdx_cap = 1;
3516 } else {
3517 ixgbe->param_en_10000fdx_cap = 0;
3518 ixgbe->param_adv_10000fdx_cap = 0;
3519 }
3520
3521 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3522 ixgbe->param_en_5000fdx_cap = 1;
3523 ixgbe->param_adv_5000fdx_cap = 1;
3524 } else {
3525 ixgbe->param_en_5000fdx_cap = 0;
3526 ixgbe->param_adv_5000fdx_cap = 0;
3527 }
3528
3529 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3530 ixgbe->param_en_2500fdx_cap = 1;
3531 ixgbe->param_adv_2500fdx_cap = 1;
3532 } else {
3533 ixgbe->param_en_2500fdx_cap = 0;
3534 ixgbe->param_adv_2500fdx_cap = 0;
3535 }
3536
3537 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3538 ixgbe->param_en_1000fdx_cap = 1;
3539 ixgbe->param_adv_1000fdx_cap = 1;
3540 } else {
3541 ixgbe->param_en_1000fdx_cap = 0;
3542 ixgbe->param_adv_1000fdx_cap = 0;
3543 }
3544
3545 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3546 ixgbe->param_en_100fdx_cap = 1;
3547 ixgbe->param_adv_100fdx_cap = 1;
3548 } else {
3549 ixgbe->param_en_100fdx_cap = 0;
3550 ixgbe->param_adv_100fdx_cap = 0;
3551 }
3552
3553 ixgbe->param_pause_cap = 1;
3554 ixgbe->param_asym_pause_cap = 1;
3555 ixgbe->param_rem_fault = 0;
3556
3557 ixgbe->param_adv_autoneg_cap = 1;
3558 ixgbe->param_adv_pause_cap = 1;
3559 ixgbe->param_adv_asym_pause_cap = 1;
3560 ixgbe->param_adv_rem_fault = 0;
3561
3562 ixgbe->param_lp_10000fdx_cap = 0;
3563 ixgbe->param_lp_5000fdx_cap = 0;
3564 ixgbe->param_lp_2500fdx_cap = 0;
3565 ixgbe->param_lp_1000fdx_cap = 0;
3566 ixgbe->param_lp_100fdx_cap = 0;
3567 ixgbe->param_lp_autoneg_cap = 0;
3568 ixgbe->param_lp_pause_cap = 0;
3569 ixgbe->param_lp_asym_pause_cap = 0;
3570 ixgbe->param_lp_rem_fault = 0;
3571 }
3572
3573 /*
3574 * ixgbe_get_prop - Get a property value out of the configuration file
3575 * ixgbe.conf.
3576 *
3577 * Caller provides the name of the property, a default value, a minimum
3578 * value, and a maximum value.
3579 *
3580 * Return configured value of the property, with default, minimum and
3581 * maximum properly applied.
3582 */
3583 static int
3584 ixgbe_get_prop(ixgbe_t *ixgbe,
3592 /*
3593 * Call ddi_prop_get_int() to read the conf settings
3594 */
3595 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3596 DDI_PROP_DONTPASS, propname, defval);
3597 if (value > maxval)
3598 value = maxval;
3599
3600 if (value < minval)
3601 value = minval;
3602
3603 return (value);
3604 }
3605
3606 /*
3607 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3608 */
3609 int
3610 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3611 {
3612 struct ixgbe_hw *hw = &ixgbe->hw;
3613 ixgbe_link_speed advertised = 0;
3614
3615 /*
3616 * Assemble a list of enabled speeds to auto-negotiate with.
3617 */
3618 if (ixgbe->param_en_10000fdx_cap == 1)
3619 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3620
3621 if (ixgbe->param_en_5000fdx_cap == 1)
3622 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3623
3624 if (ixgbe->param_en_2500fdx_cap == 1)
3625 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3626
3627 if (ixgbe->param_en_1000fdx_cap == 1)
3628 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3629
3630 if (ixgbe->param_en_100fdx_cap == 1)
3631 advertised |= IXGBE_LINK_SPEED_100_FULL;
3632
3633 /*
3634 * As a last resort, autoneg with a default list of speeds.
3635 */
3636 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3637 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3638 "to autonegotiate with full capabilities.");
3639
3640 if (hw->mac.type == ixgbe_mac_82598EB)
3641 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3642 else
3643 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3644 }
3645
3646 if (setup_hw) {
3647 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3648 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3649 ixgbe_notice(ixgbe, "Setup link failed on this "
3650 "device.");
3651 return (IXGBE_FAILURE);
3652 }
3653 }
3654
3655 return (IXGBE_SUCCESS);
3656 }
3657
3658 /*
3659 * ixgbe_driver_link_check - Link status processing.
3660 *
3661 * This function can be called in both kernel context and interrupt context
3662 */
3663 static void
3664 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3665 {
3666 struct ixgbe_hw *hw = &ixgbe->hw;
3667 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3668 boolean_t link_up = B_FALSE;
3669 boolean_t link_changed = B_FALSE;
3670
3671 ASSERT(mutex_owned(&ixgbe->gen_lock));
3672
3673 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3674 if (link_up) {
3675 ixgbe->link_check_complete = B_TRUE;
3676
3677 /* Link is up, enable flow control settings */
3678 (void) ixgbe_fc_enable(hw);
3679
3680 /*
3681 * The Link is up, check whether it was marked as down earlier
3682 */
3683 if (ixgbe->link_state != LINK_STATE_UP) {
3684 switch (speed) {
3685 case IXGBE_LINK_SPEED_10GB_FULL:
3686 ixgbe->link_speed = SPEED_10GB;
3687 break;
3688 case IXGBE_LINK_SPEED_5GB_FULL:
3689 ixgbe->link_speed = SPEED_5GB;
3690 break;
3691 case IXGBE_LINK_SPEED_2_5GB_FULL:
3692 ixgbe->link_speed = SPEED_2_5GB;
3693 break;
3694 case IXGBE_LINK_SPEED_1GB_FULL:
3695 ixgbe->link_speed = SPEED_1GB;
3696 break;
3697 case IXGBE_LINK_SPEED_100_FULL:
3698 ixgbe->link_speed = SPEED_100;
3699 }
3700 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3701 ixgbe->link_state = LINK_STATE_UP;
3702 link_changed = B_TRUE;
3703 }
3704 } else {
3705 if (ixgbe->link_check_complete == B_TRUE ||
3706 (ixgbe->link_check_complete == B_FALSE &&
3707 gethrtime() >= ixgbe->link_check_hrtime)) {
3708 /*
3709 * The link is really down
3710 */
3711 ixgbe->link_check_complete = B_TRUE;
3712
3713 if (ixgbe->link_state != LINK_STATE_DOWN) {
3727 ixgbe->eims |= IXGBE_EICR_LSC;
3728 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3729 }
3730
3731 if (link_changed) {
3732 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3733 }
3734 }
3735
3736 /*
3737 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3738 */
3739 static void
3740 ixgbe_sfp_check(void *arg)
3741 {
3742 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3743 uint32_t eicr = ixgbe->eicr;
3744 struct ixgbe_hw *hw = &ixgbe->hw;
3745
3746 mutex_enter(&ixgbe->gen_lock);
3747 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3748 /* clear the interrupt */
3749 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3750
3751 /* if link up, do multispeed fiber setup */
3752 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3753 B_TRUE);
3754 ixgbe_driver_link_check(ixgbe);
3755 ixgbe_get_hw_state(ixgbe);
3756 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3757 /* clear the interrupt */
3758 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3759
3760 /* if link up, do sfp module setup */
3761 (void) hw->mac.ops.setup_sfp(hw);
3762
3763 /* do multispeed fiber setup */
3764 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3765 B_TRUE);
3766 ixgbe_driver_link_check(ixgbe);
3767 ixgbe_get_hw_state(ixgbe);
3768 }
3769 mutex_exit(&ixgbe->gen_lock);
3770
3771 /*
3772 * We need to fully re-check the link later.
3773 */
3774 ixgbe->link_check_complete = B_FALSE;
3775 ixgbe->link_check_hrtime = gethrtime() +
3776 (IXGBE_LINK_UP_TIME * 100000000ULL);
3777 }
3778
3779 /*
3780 * ixgbe_overtemp_check - overtemp module processing done in taskq
3781 *
3782 * This routine will only be called on adapters with temperature sensor.
3783 * The indication of over-temperature can be either SDP0 interrupt or the link
3784 * status change interrupt.
3785 */
3786 static void
3787 ixgbe_overtemp_check(void *arg)
3788 {
3789 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3790 struct ixgbe_hw *hw = &ixgbe->hw;
3791 uint32_t eicr = ixgbe->eicr;
3792 ixgbe_link_speed speed;
3793 boolean_t link_up;
3794
3795 mutex_enter(&ixgbe->gen_lock);
3796
3797 /* make sure we know current state of link */
3798 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3799
3800 /* check over-temp condition */
3801 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3802 (eicr & IXGBE_EICR_LSC)) {
3803 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3804 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3805
3806 /*
3807 * Disable the adapter interrupts
3808 */
3809 ixgbe_disable_adapter_interrupts(ixgbe);
3810
3811 /*
3812 * Disable Rx/Tx units
3813 */
3814 (void) ixgbe_stop_adapter(hw);
3815
3816 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3817 ixgbe_error(ixgbe,
3818 "Problem: Network adapter has been stopped "
3819 "because it has overheated");
3820 ixgbe_error(ixgbe,
3821 "Action: Restart the computer. "
3822 "If the problem persists, power off the system "
3823 "and replace the adapter");
3824 }
3825 }
3826
3827 /* write to clear the interrupt */
3828 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3829
3830 mutex_exit(&ixgbe->gen_lock);
3831 }
3832
3833 /*
3834 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3835 *
3836 * This routine will only be called on adapters with external PHYs
3837 * (such as X550) that may be trying to raise our attention to some event.
3838 * Currently, this is limited to claiming PHY overtemperature and link status
3839 * change (LSC) events, however this may expand to include other things in
3840 * future adapters.
3841 */
3842 static void
3843 ixgbe_phy_check(void *arg)
3844 {
3845 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3846 struct ixgbe_hw *hw = &ixgbe->hw;
3847 int rv;
3848
3849 mutex_enter(&ixgbe->gen_lock);
3850
3851 /*
3852 * X550 baseT PHY overtemp and LSC events are handled here.
3853 *
3854 * If an overtemp event occurs, it will be reflected in the
3855 * return value of phy.ops.handle_lasi() and the common code will
3856 * automatically power off the baseT PHY. This is our cue to trigger
3857 * an FMA event.
3858 *
3859 * If a link status change event occurs, phy.ops.handle_lasi() will
3860 * automatically initiate a link setup between the integrated KR PHY
3861 * and the external X557 PHY to ensure that the link speed between
3862 * them matches the link speed of the baseT link.
3863 */
3864 rv = ixgbe_handle_lasi(hw);
3865
3866 if (rv == IXGBE_ERR_OVERTEMP) {
3867 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3868
3869 /*
3870 * Disable the adapter interrupts
3871 */
3872 ixgbe_disable_adapter_interrupts(ixgbe);
3873
3874 /*
3875 * Disable Rx/Tx units
3876 */
3877 (void) ixgbe_stop_adapter(hw);
3878
3879 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3880 ixgbe_error(ixgbe,
3881 "Problem: Network adapter has been stopped due to a "
3882 "overtemperature event being detected.");
3883 ixgbe_error(ixgbe,
3884 "Action: Shut down or restart the computer. If the issue "
3885 "persists, please take action in accordance with the "
3886 "recommendations from your system vendor.");
3887 }
3888
3889 mutex_exit(&ixgbe->gen_lock);
3890 }
3891
3892 /*
3893 * ixgbe_link_timer - timer for link status detection
3894 */
3895 static void
3896 ixgbe_link_timer(void *arg)
3897 {
3898 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3899
3900 mutex_enter(&ixgbe->gen_lock);
3901 ixgbe_driver_link_check(ixgbe);
3902 mutex_exit(&ixgbe->gen_lock);
3903 }
3904
3905 /*
3906 * ixgbe_local_timer - Driver watchdog function.
3907 *
3908 * This function will handle the transmit stall check and other routines.
3909 */
3910 static void
3911 ixgbe_local_timer(void *arg)
3912 {
4036
4037 /*
4038 * Look up the OBP property "local-mac-address?". If the user has set
4039 * 'local-mac-address? = false', use "the system address" instead.
4040 */
4041 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4042 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4043 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4044 if (localetheraddr(NULL, &sysaddr) != 0) {
4045 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4046 found = B_TRUE;
4047 }
4048 }
4049 ddi_prop_free(bytes);
4050 }
4051
4052 /*
4053 * Finally(!), if there's a valid "mac-address" property (created
4054 * if we netbooted from this interface), we must use this instead
4055 * of any of the above to ensure that the NFS/install server doesn't
4056 * get confused by the address changing as illumos takes over!
4057 */
4058 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4059 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4060 if (err == DDI_PROP_SUCCESS) {
4061 if (nelts == ETHERADDRL) {
4062 while (nelts--)
4063 hw->mac.addr[nelts] = bytes[nelts];
4064 found = B_TRUE;
4065 }
4066 ddi_prop_free(bytes);
4067 }
4068
4069 if (found) {
4070 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4071 return (B_TRUE);
4072 }
4073 #else
4074 _NOTE(ARGUNUSED(ixgbe));
4075 #endif
4076
4226 */
4227 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4228 /* enable autoclear but not on bits 29:20 */
4229 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4230
4231 /* general purpose interrupt enable */
4232 gpie |= (IXGBE_GPIE_MSIX_MODE
4233 | IXGBE_GPIE_PBA_SUPPORT
4234 | IXGBE_GPIE_OCD
4235 | IXGBE_GPIE_EIAME);
4236 /*
4237 * non-msi-x mode
4238 */
4239 } else {
4240
4241 /* disable autoclear, leave gpie at default */
4242 eiac = 0;
4243
4244 /*
4245 * General purpose interrupt enable.
4246 * For 82599, X540 and X550, extended interrupt
4247 * automask enable only in MSI or MSI-X mode
4248 */
4249 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4250 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4251 gpie |= IXGBE_GPIE_EIAME;
4252 }
4253 }
4254
4255 /* Enable specific "other" interrupt types */
4256 switch (hw->mac.type) {
4257 case ixgbe_mac_82598EB:
4258 gpie |= ixgbe->capab->other_gpie;
4259 break;
4260
4261 case ixgbe_mac_82599EB:
4262 case ixgbe_mac_X540:
4263 case ixgbe_mac_X550:
4264 case ixgbe_mac_X550EM_x:
4265 gpie |= ixgbe->capab->other_gpie;
4266
4267 /* Enable RSC Delay 8us when LRO enabled */
4268 if (ixgbe->lro_enable) {
4269 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4270 }
4271 break;
4272
4273 default:
4274 break;
4275 }
4276
4277 /* write to interrupt control registers */
4278 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4279 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4280 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4281 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4282 IXGBE_WRITE_FLUSH(hw);
4283 }
4284
4439 &atlas);
4440 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4441 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4442 atlas);
4443
4444 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4445 &atlas);
4446 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4447 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4448 atlas);
4449
4450 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4451 &atlas);
4452 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4453 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4454 atlas);
4455 break;
4456
4457 case ixgbe_mac_82599EB:
4458 case ixgbe_mac_X540:
4459 case ixgbe_mac_X550:
4460 case ixgbe_mac_X550EM_x:
4461 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4462 reg |= (IXGBE_AUTOC_FLU |
4463 IXGBE_AUTOC_10G_KX4);
4464 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4465
4466 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4467 B_FALSE);
4468 break;
4469
4470 default:
4471 break;
4472 }
4473 }
4474
4475 #pragma inline(ixgbe_intr_rx_work)
4476 /*
4477 * ixgbe_intr_rx_work - RX processing of ISR.
4478 */
4479 static void
4480 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4481 {
4482 mblk_t *mp;
4483
4484 mutex_enter(&rx_ring->rx_lock);
4485
4486 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4487 mutex_exit(&rx_ring->rx_lock);
4507
4508 /*
4509 * Schedule the re-transmit
4510 */
4511 if (tx_ring->reschedule &&
4512 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4513 tx_ring->reschedule = B_FALSE;
4514 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4515 tx_ring->ring_handle);
4516 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4517 }
4518 }
4519
4520 #pragma inline(ixgbe_intr_other_work)
4521 /*
4522 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4523 */
4524 static void
4525 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4526 {
4527 struct ixgbe_hw *hw = &ixgbe->hw;
4528
4529 ASSERT(mutex_owned(&ixgbe->gen_lock));
4530
4531 /*
4532 * handle link status change
4533 */
4534 if (eicr & IXGBE_EICR_LSC) {
4535 ixgbe_driver_link_check(ixgbe);
4536 ixgbe_get_hw_state(ixgbe);
4537 }
4538
4539 /*
4540 * check for fan failure on adapters with fans
4541 */
4542 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4543 (eicr & IXGBE_EICR_GPI_SDP1)) {
4544 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4545
4546 /*
4547 * Disable the adapter interrupts
4548 */
4551 /*
4552 * Disable Rx/Tx units
4553 */
4554 (void) ixgbe_stop_adapter(&ixgbe->hw);
4555
4556 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4557 ixgbe_error(ixgbe,
4558 "Problem: Network adapter has been stopped "
4559 "because the fan has stopped.\n");
4560 ixgbe_error(ixgbe,
4561 "Action: Replace the adapter.\n");
4562
4563 /* re-enable the interrupt, which was automasked */
4564 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4565 }
4566
4567 /*
4568 * Do SFP check for adapters with hot-plug capability
4569 */
4570 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4571 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4572 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4573 ixgbe->eicr = eicr;
4574 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4575 ixgbe_sfp_check, (void *)ixgbe,
4576 DDI_NOSLEEP)) != DDI_SUCCESS) {
4577 ixgbe_log(ixgbe, "No memory available to dispatch "
4578 "taskq for SFP check");
4579 }
4580 }
4581
4582 /*
4583 * Do over-temperature check for adapters with temp sensor
4584 */
4585 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4586 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4587 (eicr & IXGBE_EICR_LSC))) {
4588 ixgbe->eicr = eicr;
4589 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4590 ixgbe_overtemp_check, (void *)ixgbe,
4591 DDI_NOSLEEP)) != DDI_SUCCESS) {
4592 ixgbe_log(ixgbe, "No memory available to dispatch "
4593 "taskq for overtemp check");
4594 }
4595 }
4596
4597 /*
4598 * Process an external PHY interrupt
4599 */
4600 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4601 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4602 ixgbe->eicr = eicr;
4603 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4604 ixgbe_phy_check, (void *)ixgbe,
4605 DDI_NOSLEEP)) != DDI_SUCCESS) {
4606 ixgbe_log(ixgbe, "No memory available to dispatch "
4607 "taskq for PHY check");
4608 }
4609 }
4610 }
4611
4612 /*
4613 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4614 */
4615 static uint_t
4616 ixgbe_intr_legacy(void *arg1, void *arg2)
4617 {
4618 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4619 struct ixgbe_hw *hw = &ixgbe->hw;
4620 ixgbe_tx_ring_t *tx_ring;
4621 ixgbe_rx_ring_t *rx_ring;
4622 uint32_t eicr;
4623 mblk_t *mp;
4624 boolean_t tx_reschedule;
4625 uint_t result;
4626
4627 _NOTE(ARGUNUSED(arg2));
4628
4629 mutex_enter(&ixgbe->gen_lock);
4678 */
4679 tx_ring = &ixgbe->tx_rings[0];
4680 tx_ring->tx_recycle(tx_ring);
4681
4682 /*
4683 * Schedule the re-transmit
4684 */
4685 tx_reschedule = (tx_ring->reschedule &&
4686 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4687 }
4688
4689 /* any interrupt type other than tx/rx */
4690 if (eicr & ixgbe->capab->other_intr) {
4691 switch (hw->mac.type) {
4692 case ixgbe_mac_82598EB:
4693 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4694 break;
4695
4696 case ixgbe_mac_82599EB:
4697 case ixgbe_mac_X540:
4698 case ixgbe_mac_X550:
4699 case ixgbe_mac_X550EM_x:
4700 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4701 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4702 break;
4703
4704 default:
4705 break;
4706 }
4707 ixgbe_intr_other_work(ixgbe, eicr);
4708 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4709 }
4710
4711 mutex_exit(&ixgbe->gen_lock);
4712
4713 result = DDI_INTR_CLAIMED;
4714 } else {
4715 mutex_exit(&ixgbe->gen_lock);
4716
4717 /*
4718 * No interrupt cause bits set: don't claim this interrupt.
4719 */
4774 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4775 }
4776
4777 /*
4778 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4779 */
4780 if (eicr & 0x2) {
4781 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4782 }
4783
4784 /* any interrupt type other than tx/rx */
4785 if (eicr & ixgbe->capab->other_intr) {
4786 mutex_enter(&ixgbe->gen_lock);
4787 switch (hw->mac.type) {
4788 case ixgbe_mac_82598EB:
4789 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4790 break;
4791
4792 case ixgbe_mac_82599EB:
4793 case ixgbe_mac_X540:
4794 case ixgbe_mac_X550:
4795 case ixgbe_mac_X550EM_x:
4796 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4797 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4798 break;
4799
4800 default:
4801 break;
4802 }
4803 ixgbe_intr_other_work(ixgbe, eicr);
4804 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4805 mutex_exit(&ixgbe->gen_lock);
4806 }
4807
4808 /* re-enable the interrupts which were automasked */
4809 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4810
4811 return (DDI_INTR_CLAIMED);
4812 }
4813
4814 /*
4815 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4856 DDI_FM_OK) {
4857 ddi_fm_service_impact(ixgbe->dip,
4858 DDI_SERVICE_DEGRADED);
4859 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4860 return (DDI_INTR_CLAIMED);
4861 }
4862
4863 /*
4864 * Check "other" cause bits: any interrupt type other than tx/rx
4865 */
4866 if (eicr & ixgbe->capab->other_intr) {
4867 mutex_enter(&ixgbe->gen_lock);
4868 switch (hw->mac.type) {
4869 case ixgbe_mac_82598EB:
4870 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4871 ixgbe_intr_other_work(ixgbe, eicr);
4872 break;
4873
4874 case ixgbe_mac_82599EB:
4875 case ixgbe_mac_X540:
4876 case ixgbe_mac_X550:
4877 case ixgbe_mac_X550EM_x:
4878 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4879 ixgbe_intr_other_work(ixgbe, eicr);
4880 break;
4881
4882 default:
4883 break;
4884 }
4885 mutex_exit(&ixgbe->gen_lock);
4886 }
4887
4888 /* re-enable the interrupts which were automasked */
4889 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4890 }
4891
4892 return (DDI_INTR_CLAIMED);
4893 }
4894
4895 /*
4896 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4897 *
4947 ixgbe_log(ixgbe,
4948 "MSI-X not used, force rings and groups number to 1");
4949
4950 /*
4951 * Install MSI interrupts
4952 */
4953 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4954 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4955 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4956 if (rc == IXGBE_SUCCESS)
4957 return (IXGBE_SUCCESS);
4958
4959 ixgbe_log(ixgbe,
4960 "Allocate MSI failed, trying Legacy interrupts...");
4961 }
4962
4963 /*
4964 * Install legacy interrupts
4965 */
4966 if (intr_types & DDI_INTR_TYPE_FIXED) {
4967 /*
4968 * Disallow legacy interrupts for X550. X550 has a silicon
4969 * bug which prevents Shared Legacy interrupts from working.
4970 * For details, please reference:
4971 *
4972 * Intel Ethernet Controller X550 Specification Update rev. 2.1
4973 * May 2016, erratum 22: PCIe Interrupt Status Bit
4974 */
4975 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
4976 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
4977 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
4978 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf) {
4979 ixgbe_log(ixgbe,
4980 "Legacy interrupts are not supported on this "
4981 "adapter. Please use MSI or MSI-X instead.");
4982 return (IXGBE_FAILURE);
4983 }
4984 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4985 if (rc == IXGBE_SUCCESS)
4986 return (IXGBE_SUCCESS);
4987
4988 ixgbe_log(ixgbe,
4989 "Allocate Legacy interrupts failed");
4990 }
4991
4992 /*
4993 * If none of the 3 types succeeded, return failure
4994 */
4995 return (IXGBE_FAILURE);
4996 }
4997
4998 /*
4999 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5000 *
5001 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5002 * if fewer than 2 handles are available, return failure.
5003 * Upon success, this maps the vectors to rx and tx rings for
5276 int8_t cause)
5277 {
5278 struct ixgbe_hw *hw = &ixgbe->hw;
5279 u32 ivar, index;
5280
5281 switch (hw->mac.type) {
5282 case ixgbe_mac_82598EB:
5283 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5284 if (cause == -1) {
5285 cause = 0;
5286 }
5287 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5288 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5289 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5290 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5291 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5292 break;
5293
5294 case ixgbe_mac_82599EB:
5295 case ixgbe_mac_X540:
5296 case ixgbe_mac_X550:
5297 case ixgbe_mac_X550EM_x:
5298 if (cause == -1) {
5299 /* other causes */
5300 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5301 index = (intr_alloc_entry & 1) * 8;
5302 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5303 ivar &= ~(0xFF << index);
5304 ivar |= (msix_vector << index);
5305 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5306 } else {
5307 /* tx or rx causes */
5308 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5309 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5310 ivar = IXGBE_READ_REG(hw,
5311 IXGBE_IVAR(intr_alloc_entry >> 1));
5312 ivar &= ~(0xFF << index);
5313 ivar |= (msix_vector << index);
5314 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5315 ivar);
5316 }
5317 break;
5332 static void
5333 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5334 {
5335 struct ixgbe_hw *hw = &ixgbe->hw;
5336 u32 ivar, index;
5337
5338 switch (hw->mac.type) {
5339 case ixgbe_mac_82598EB:
5340 if (cause == -1) {
5341 cause = 0;
5342 }
5343 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5344 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5345 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5346 (intr_alloc_entry & 0x3)));
5347 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5348 break;
5349
5350 case ixgbe_mac_82599EB:
5351 case ixgbe_mac_X540:
5352 case ixgbe_mac_X550:
5353 case ixgbe_mac_X550EM_x:
5354 if (cause == -1) {
5355 /* other causes */
5356 index = (intr_alloc_entry & 1) * 8;
5357 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5358 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5359 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5360 } else {
5361 /* tx or rx causes */
5362 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5363 ivar = IXGBE_READ_REG(hw,
5364 IXGBE_IVAR(intr_alloc_entry >> 1));
5365 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5366 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5367 ivar);
5368 }
5369 break;
5370
5371 default:
5372 break;
5373 }
5384 static void
5385 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5386 {
5387 struct ixgbe_hw *hw = &ixgbe->hw;
5388 u32 ivar, index;
5389
5390 switch (hw->mac.type) {
5391 case ixgbe_mac_82598EB:
5392 if (cause == -1) {
5393 cause = 0;
5394 }
5395 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5396 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5397 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5398 (intr_alloc_entry & 0x3)));
5399 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5400 break;
5401
5402 case ixgbe_mac_82599EB:
5403 case ixgbe_mac_X540:
5404 case ixgbe_mac_X550:
5405 case ixgbe_mac_X550EM_x:
5406 if (cause == -1) {
5407 /* other causes */
5408 index = (intr_alloc_entry & 1) * 8;
5409 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5410 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5411 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5412 } else {
5413 /* tx or rx causes */
5414 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5415 ivar = IXGBE_READ_REG(hw,
5416 IXGBE_IVAR(intr_alloc_entry >> 1));
5417 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5418 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5419 ivar);
5420 }
5421 break;
5422
5423 default:
5424 break;
5425 }
5429 * Convert the rx ring index driver maintained to the rx ring index
5430 * in h/w.
5431 */
5432 static uint32_t
5433 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5434 {
5435
5436 struct ixgbe_hw *hw = &ixgbe->hw;
5437 uint32_t rx_ring_per_group, hw_rx_index;
5438
5439 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5440 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5441 return (sw_rx_index);
5442 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5443 switch (hw->mac.type) {
5444 case ixgbe_mac_82598EB:
5445 return (sw_rx_index);
5446
5447 case ixgbe_mac_82599EB:
5448 case ixgbe_mac_X540:
5449 case ixgbe_mac_X550:
5450 case ixgbe_mac_X550EM_x:
5451 return (sw_rx_index * 2);
5452
5453 default:
5454 break;
5455 }
5456 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5457 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5458
5459 switch (hw->mac.type) {
5460 case ixgbe_mac_82598EB:
5461 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5462 16 + (sw_rx_index % rx_ring_per_group);
5463 return (hw_rx_index);
5464
5465 case ixgbe_mac_82599EB:
5466 case ixgbe_mac_X540:
5467 case ixgbe_mac_X550:
5468 case ixgbe_mac_X550EM_x:
5469 if (ixgbe->num_rx_groups > 32) {
5470 hw_rx_index = (sw_rx_index /
5471 rx_ring_per_group) * 2 +
5472 (sw_rx_index % rx_ring_per_group);
5473 } else {
5474 hw_rx_index = (sw_rx_index /
5475 rx_ring_per_group) * 4 +
5476 (sw_rx_index % rx_ring_per_group);
5477 }
5478 return (hw_rx_index);
5479
5480 default:
5481 break;
5482 }
5483 }
5484
5485 /*
5486 * Should never reach. Just to make compiler happy.
5487 */
5488 return (sw_rx_index);
5554 static void
5555 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5556 {
5557 struct ixgbe_hw *hw = &ixgbe->hw;
5558 ixgbe_intr_vector_t *vect; /* vector bitmap */
5559 int r_idx; /* ring index */
5560 int v_idx; /* vector index */
5561 uint32_t hw_index;
5562
5563 /*
5564 * Clear any previous entries
5565 */
5566 switch (hw->mac.type) {
5567 case ixgbe_mac_82598EB:
5568 for (v_idx = 0; v_idx < 25; v_idx++)
5569 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5570 break;
5571
5572 case ixgbe_mac_82599EB:
5573 case ixgbe_mac_X540:
5574 case ixgbe_mac_X550:
5575 case ixgbe_mac_X550EM_x:
5576 for (v_idx = 0; v_idx < 64; v_idx++)
5577 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5578 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5579 break;
5580
5581 default:
5582 break;
5583 }
5584
5585 /*
5586 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5587 * tx rings[0] will use RTxQ[1].
5588 */
5589 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5590 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5591 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5592 return;
5593 }
5594
5595 /*
5731 for (i = 0; i < ixgbe->intr_cnt; i++) {
5732 rc = ddi_intr_disable(ixgbe->htable[i]);
5733 if (rc != DDI_SUCCESS) {
5734 ixgbe_log(ixgbe,
5735 "Disable intr failed: %d", rc);
5736 return (IXGBE_FAILURE);
5737 }
5738 }
5739 }
5740
5741 return (IXGBE_SUCCESS);
5742 }
5743
5744 /*
5745 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5746 */
5747 static void
5748 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5749 {
5750 struct ixgbe_hw *hw = &ixgbe->hw;
5751 ixgbe_link_speed speed = 0;
5752 boolean_t link_up = B_FALSE;
5753 uint32_t pcs1g_anlp = 0;
5754
5755 ASSERT(mutex_owned(&ixgbe->gen_lock));
5756 ixgbe->param_lp_1000fdx_cap = 0;
5757 ixgbe->param_lp_100fdx_cap = 0;
5758
5759 /* check for link, don't wait */
5760 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5761
5762 /*
5763 * Update the observed Link Partner's capabilities. Not all adapters
5764 * can provide full information on the LP's capable speeds, so we
5765 * provide what we can.
5766 */
5767 if (link_up) {
5768 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5769
5770 ixgbe->param_lp_1000fdx_cap =
5771 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5772 ixgbe->param_lp_100fdx_cap =
5773 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5774 }
5775
5776 /*
5777 * Update GLD's notion of the adapter's currently advertised speeds.
5778 * Since the common code doesn't always record the current autonegotiate
5779 * settings in the phy struct for all parts (specifically, adapters with
5780 * SFPs) we first test to see if it is 0, and if so, we fall back to
5781 * using the adapter's speed capabilities which we saved during instance
5782 * init in ixgbe_init_params().
5783 *
5784 * Adapters with SFPs will always be shown as advertising all of their
5785 * supported speeds, and adapters with baseT PHYs (where the phy struct
5786 * is maintained by the common code) will always have a factual view of
5787 * their currently-advertised speeds. In the case of SFPs, this is
5788 * acceptable as we default to advertising all speeds that the adapter
5789 * claims to support, and those properties are immutable; unlike on
5790 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5791 */
5792 speed = hw->phy.autoneg_advertised;
5793 if (speed == 0)
5794 speed = ixgbe->speeds_supported;
5795
5796 ixgbe->param_adv_10000fdx_cap =
5797 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5798 ixgbe->param_adv_5000fdx_cap =
5799 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5800 ixgbe->param_adv_2500fdx_cap =
5801 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5802 ixgbe->param_adv_1000fdx_cap =
5803 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5804 ixgbe->param_adv_100fdx_cap =
5805 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5806 }
5807
5808 /*
5809 * ixgbe_get_driver_control - Notify that driver is in control of device.
5810 */
5811 static void
5812 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5813 {
5814 uint32_t ctrl_ext;
5815
5816 /*
5817 * Notify firmware that driver is in control of device
5818 */
5819 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5820 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5821 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5822 }
5823
5824 /*
5825 * ixgbe_release_driver_control - Notify that driver is no longer in control
|