1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38
39 /*
40 * Local function protoypes
41 */
42 static int ixgbe_register_mac(ixgbe_t *);
43 static int ixgbe_identify_hardware(ixgbe_t *);
44 static int ixgbe_regs_map(ixgbe_t *);
45 static void ixgbe_init_properties(ixgbe_t *);
46 static int ixgbe_init_driver_settings(ixgbe_t *);
47 static void ixgbe_init_locks(ixgbe_t *);
48 static void ixgbe_destroy_locks(ixgbe_t *);
49 static int ixgbe_init(ixgbe_t *);
50 static int ixgbe_chip_start(ixgbe_t *);
51 static void ixgbe_chip_stop(ixgbe_t *);
52 static int ixgbe_reset(ixgbe_t *);
53 static void ixgbe_tx_clean(ixgbe_t *);
54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
56 static int ixgbe_alloc_rings(ixgbe_t *);
57 static void ixgbe_free_rings(ixgbe_t *);
58 static int ixgbe_alloc_rx_data(ixgbe_t *);
59 static void ixgbe_free_rx_data(ixgbe_t *);
60 static void ixgbe_setup_rings(ixgbe_t *);
61 static void ixgbe_setup_rx(ixgbe_t *);
62 static void ixgbe_setup_tx(ixgbe_t *);
63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
65 static void ixgbe_setup_rss(ixgbe_t *);
66 static void ixgbe_setup_vmdq(ixgbe_t *);
67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
68 static void ixgbe_setup_rss_table(ixgbe_t *);
69 static void ixgbe_init_unicst(ixgbe_t *);
70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
71 static void ixgbe_setup_multicst(ixgbe_t *);
72 static void ixgbe_get_hw_state(ixgbe_t *);
73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
74 static void ixgbe_get_conf(ixgbe_t *);
75 static void ixgbe_init_params(ixgbe_t *);
76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
77 static void ixgbe_driver_link_check(ixgbe_t *);
78 static void ixgbe_sfp_check(void *);
79 static void ixgbe_overtemp_check(void *);
80 static void ixgbe_phy_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
104 static void ixgbe_rem_intrs(ixgbe_t *);
105 static int ixgbe_enable_intrs(ixgbe_t *);
106 static int ixgbe_disable_intrs(ixgbe_t *);
107 static uint_t ixgbe_intr_legacy(void *, void *);
108 static uint_t ixgbe_intr_msi(void *, void *);
109 static uint_t ixgbe_intr_msix(void *, void *);
110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
114 static int ixgbe_addmac(void *, const uint8_t *);
115 static int ixgbe_remmac(void *, const uint8_t *);
116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
117
118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
120 static int ixgbe_resume(dev_info_t *);
121 static int ixgbe_suspend(dev_info_t *);
122 static int ixgbe_quiesce(dev_info_t *);
123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
126 static int ixgbe_intr_cb_register(ixgbe_t *);
127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
128
129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
130 const void *impl_data);
131 static void ixgbe_fm_init(ixgbe_t *);
132 static void ixgbe_fm_fini(ixgbe_t *);
133
134 char *ixgbe_priv_props[] = {
135 "_tx_copy_thresh",
136 "_tx_recycle_thresh",
137 "_tx_overload_thresh",
138 "_tx_resched_thresh",
139 "_rx_copy_thresh",
140 "_rx_limit_per_intr",
141 "_intr_throttling",
142 "_adv_pause_cap",
143 "_adv_asym_pause_cap",
144 NULL
145 };
146
147 #define IXGBE_MAX_PRIV_PROPS \
148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
149
150 static struct cb_ops ixgbe_cb_ops = {
151 nulldev, /* cb_open */
152 nulldev, /* cb_close */
153 nodev, /* cb_strategy */
154 nodev, /* cb_print */
155 nodev, /* cb_dump */
156 nodev, /* cb_read */
157 nodev, /* cb_write */
158 nodev, /* cb_ioctl */
159 nodev, /* cb_devmap */
160 nodev, /* cb_mmap */
161 nodev, /* cb_segmap */
162 nochpoll, /* cb_chpoll */
163 ddi_prop_op, /* cb_prop_op */
164 NULL, /* cb_stream */
165 D_MP | D_HOTPLUG, /* cb_flag */
166 CB_REV, /* cb_rev */
167 nodev, /* cb_aread */
168 nodev /* cb_awrite */
169 };
170
171 static struct dev_ops ixgbe_dev_ops = {
172 DEVO_REV, /* devo_rev */
173 0, /* devo_refcnt */
174 NULL, /* devo_getinfo */
175 nulldev, /* devo_identify */
176 nulldev, /* devo_probe */
177 ixgbe_attach, /* devo_attach */
178 ixgbe_detach, /* devo_detach */
179 nodev, /* devo_reset */
180 &ixgbe_cb_ops, /* devo_cb_ops */
181 NULL, /* devo_bus_ops */
182 ddi_power, /* devo_power */
183 ixgbe_quiesce, /* devo_quiesce */
184 };
185
186 static struct modldrv ixgbe_modldrv = {
187 &mod_driverops, /* Type of module. This one is a driver */
188 ixgbe_ident, /* Discription string */
189 &ixgbe_dev_ops /* driver ops */
190 };
191
192 static struct modlinkage ixgbe_modlinkage = {
193 MODREV_1, &ixgbe_modldrv, NULL
194 };
195
196 /*
197 * Access attributes for register mapping
198 */
199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
200 DDI_DEVICE_ATTR_V1,
201 DDI_STRUCTURE_LE_ACC,
202 DDI_STRICTORDER_ACC,
203 DDI_FLAGERR_ACC
204 };
205
206 /*
207 * Loopback property
208 */
209 static lb_property_t lb_normal = {
210 normal, "normal", IXGBE_LB_NONE
211 };
212
213 static lb_property_t lb_mac = {
214 internal, "MAC", IXGBE_LB_INTERNAL_MAC
215 };
216
217 static lb_property_t lb_external = {
218 external, "External", IXGBE_LB_EXTERNAL
219 };
220
221 #define IXGBE_M_CALLBACK_FLAGS \
222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
223
224 static mac_callbacks_t ixgbe_m_callbacks = {
225 IXGBE_M_CALLBACK_FLAGS,
226 ixgbe_m_stat,
227 ixgbe_m_start,
228 ixgbe_m_stop,
229 ixgbe_m_promisc,
230 ixgbe_m_multicst,
231 NULL,
232 NULL,
233 NULL,
234 ixgbe_m_ioctl,
235 ixgbe_m_getcapab,
236 NULL,
237 NULL,
238 ixgbe_m_setprop,
239 ixgbe_m_getprop,
240 ixgbe_m_propinfo
241 };
242
243 /*
244 * Initialize capabilities of each supported adapter type
245 */
246 static adapter_info_t ixgbe_82598eb_cap = {
247 64, /* maximum number of rx queues */
248 1, /* minimum number of rx queues */
249 64, /* default number of rx queues */
250 16, /* maximum number of rx groups */
251 1, /* minimum number of rx groups */
252 1, /* default number of rx groups */
253 32, /* maximum number of tx queues */
254 1, /* minimum number of tx queues */
255 8, /* default number of tx queues */
256 16366, /* maximum MTU size */
257 0xFFFF, /* maximum interrupt throttle rate */
258 0, /* minimum interrupt throttle rate */
259 200, /* default interrupt throttle rate */
260 18, /* maximum total msix vectors */
261 16, /* maximum number of ring vectors */
262 2, /* maximum number of other vectors */
263 IXGBE_EICR_LSC, /* "other" interrupt types handled */
264 0, /* "other" interrupt types enable mask */
265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
266 | IXGBE_FLAG_RSS_CAPABLE
267 | IXGBE_FLAG_VMDQ_CAPABLE)
268 };
269
270 static adapter_info_t ixgbe_82599eb_cap = {
271 128, /* maximum number of rx queues */
272 1, /* minimum number of rx queues */
273 128, /* default number of rx queues */
274 64, /* maximum number of rx groups */
275 1, /* minimum number of rx groups */
276 1, /* default number of rx groups */
277 128, /* maximum number of tx queues */
278 1, /* minimum number of tx queues */
279 8, /* default number of tx queues */
280 15500, /* maximum MTU size */
281 0xFF8, /* maximum interrupt throttle rate */
282 0, /* minimum interrupt throttle rate */
283 200, /* default interrupt throttle rate */
284 64, /* maximum total msix vectors */
285 16, /* maximum number of ring vectors */
286 2, /* maximum number of other vectors */
287 (IXGBE_EICR_LSC
288 | IXGBE_EICR_GPI_SDP1
289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
290
291 (IXGBE_SDP1_GPIEN
292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
293
294 (IXGBE_FLAG_DCA_CAPABLE
295 | IXGBE_FLAG_RSS_CAPABLE
296 | IXGBE_FLAG_VMDQ_CAPABLE
297 | IXGBE_FLAG_RSC_CAPABLE
298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 0x200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 IXGBE_EICR_LSC, /* "other" interrupt types handled */
349 0, /* "other" interrupt types enable mask */
350 (IXGBE_FLAG_RSS_CAPABLE
351 | IXGBE_FLAG_VMDQ_CAPABLE
352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
353 };
354
355 /*
356 * Module Initialization Functions.
357 */
358
359 int
360 _init(void)
361 {
362 int status;
363
364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
365
366 status = mod_install(&ixgbe_modlinkage);
367
368 if (status != DDI_SUCCESS) {
369 mac_fini_ops(&ixgbe_dev_ops);
370 }
371
372 return (status);
373 }
374
375 int
376 _fini(void)
377 {
378 int status;
379
380 status = mod_remove(&ixgbe_modlinkage);
381
382 if (status == DDI_SUCCESS) {
383 mac_fini_ops(&ixgbe_dev_ops);
384 }
385
386 return (status);
387 }
388
389 int
390 _info(struct modinfo *modinfop)
391 {
392 int status;
393
394 status = mod_info(&ixgbe_modlinkage, modinfop);
395
396 return (status);
397 }
398
399 /*
400 * ixgbe_attach - Driver attach.
401 *
402 * This function is the device specific initialization entry
403 * point. This entry point is required and must be written.
404 * The DDI_ATTACH command must be provided in the attach entry
405 * point. When attach() is called with cmd set to DDI_ATTACH,
406 * all normal kernel services (such as kmem_alloc(9F)) are
407 * available for use by the driver.
408 *
409 * The attach() function will be called once for each instance
410 * of the device on the system with cmd set to DDI_ATTACH.
411 * Until attach() succeeds, the only driver entry points which
412 * may be called are open(9E) and getinfo(9E).
413 */
414 static int
415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
416 {
417 ixgbe_t *ixgbe;
418 struct ixgbe_osdep *osdep;
419 struct ixgbe_hw *hw;
420 int instance;
421 char taskqname[32];
422
423 /*
424 * Check the command and perform corresponding operations
425 */
426 switch (cmd) {
427 default:
428 return (DDI_FAILURE);
429
430 case DDI_RESUME:
431 return (ixgbe_resume(devinfo));
432
433 case DDI_ATTACH:
434 break;
435 }
436
437 /* Get the device instance */
438 instance = ddi_get_instance(devinfo);
439
440 /* Allocate memory for the instance data structure */
441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
442
443 ixgbe->dip = devinfo;
444 ixgbe->instance = instance;
445
446 hw = &ixgbe->hw;
447 osdep = &ixgbe->osdep;
448 hw->back = osdep;
449 osdep->ixgbe = ixgbe;
450
451 /* Attach the instance pointer to the dev_info data structure */
452 ddi_set_driver_private(devinfo, ixgbe);
453
454 /*
455 * Initialize for FMA support
456 */
457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
460 ixgbe_fm_init(ixgbe);
461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
462
463 /*
464 * Map PCI config space registers
465 */
466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
467 ixgbe_error(ixgbe, "Failed to map PCI configurations");
468 goto attach_fail;
469 }
470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
471
472 /*
473 * Identify the chipset family
474 */
475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
476 ixgbe_error(ixgbe, "Failed to identify hardware");
477 goto attach_fail;
478 }
479
480 /*
481 * Map device registers
482 */
483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
484 ixgbe_error(ixgbe, "Failed to map device registers");
485 goto attach_fail;
486 }
487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
488
489 /*
490 * Initialize driver parameters
491 */
492 ixgbe_init_properties(ixgbe);
493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
494
495 /*
496 * Register interrupt callback
497 */
498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
499 ixgbe_error(ixgbe, "Failed to register interrupt callback");
500 goto attach_fail;
501 }
502
503 /*
504 * Allocate interrupts
505 */
506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
507 ixgbe_error(ixgbe, "Failed to allocate interrupts");
508 goto attach_fail;
509 }
510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
511
512 /*
513 * Allocate rx/tx rings based on the ring numbers.
514 * The actual numbers of rx/tx rings are decided by the number of
515 * allocated interrupt vectors, so we should allocate the rings after
516 * interrupts are allocated.
517 */
518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
520 goto attach_fail;
521 }
522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
523
524 /*
525 * Map rings to interrupt vectors
526 */
527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
529 goto attach_fail;
530 }
531
532 /*
533 * Add interrupt handlers
534 */
535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
536 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
537 goto attach_fail;
538 }
539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
540
541 /*
542 * Create a taskq for sfp-change
543 */
544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
546 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
547 ixgbe_error(ixgbe, "sfp_taskq create failed");
548 goto attach_fail;
549 }
550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
551
552 /*
553 * Create a taskq for over-temp
554 */
555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
557 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
558 ixgbe_error(ixgbe, "overtemp_taskq create failed");
559 goto attach_fail;
560 }
561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
562
563 /*
564 * Create a taskq for processing external PHY interrupts
565 */
566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
568 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
569 ixgbe_error(ixgbe, "phy_taskq create failed");
570 goto attach_fail;
571 }
572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
573
574 /*
575 * Initialize driver parameters
576 */
577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
578 ixgbe_error(ixgbe, "Failed to initialize driver settings");
579 goto attach_fail;
580 }
581
582 /*
583 * Initialize mutexes for this device.
584 * Do this before enabling the interrupt handler and
585 * register the softint to avoid the condition where
586 * interrupt handler can try using uninitialized mutex.
587 */
588 ixgbe_init_locks(ixgbe);
589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
590
591 /*
592 * Initialize chipset hardware
593 */
594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
595 ixgbe_error(ixgbe, "Failed to initialize adapter");
596 goto attach_fail;
597 }
598 ixgbe->link_check_complete = B_FALSE;
599 ixgbe->link_check_hrtime = gethrtime() +
600 (IXGBE_LINK_UP_TIME * 100000000ULL);
601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
602
603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
605 goto attach_fail;
606 }
607
608 /*
609 * Initialize adapter capabilities
610 */
611 ixgbe_init_params(ixgbe);
612
613 /*
614 * Initialize statistics
615 */
616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
617 ixgbe_error(ixgbe, "Failed to initialize statistics");
618 goto attach_fail;
619 }
620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
621
622 /*
623 * Register the driver to the MAC
624 */
625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
626 ixgbe_error(ixgbe, "Failed to register MAC");
627 goto attach_fail;
628 }
629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
631
632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
634 if (ixgbe->periodic_id == 0) {
635 ixgbe_error(ixgbe, "Failed to add the link check timer");
636 goto attach_fail;
637 }
638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
639
640 /*
641 * Now that mutex locks are initialized, and the chip is also
642 * initialized, enable interrupts.
643 */
644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
646 goto attach_fail;
647 }
648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
649
650 ixgbe_log(ixgbe, "%s", ixgbe_ident);
651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
652
653 return (DDI_SUCCESS);
654
655 attach_fail:
656 ixgbe_unconfigure(devinfo, ixgbe);
657 return (DDI_FAILURE);
658 }
659
660 /*
661 * ixgbe_detach - Driver detach.
662 *
663 * The detach() function is the complement of the attach routine.
664 * If cmd is set to DDI_DETACH, detach() is used to remove the
665 * state associated with a given instance of a device node
666 * prior to the removal of that instance from the system.
667 *
668 * The detach() function will be called once for each instance
669 * of the device for which there has been a successful attach()
670 * once there are no longer any opens on the device.
671 *
672 * Interrupts routine are disabled, All memory allocated by this
673 * driver are freed.
674 */
675 static int
676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
677 {
678 ixgbe_t *ixgbe;
679
680 /*
681 * Check detach command
682 */
683 switch (cmd) {
684 default:
685 return (DDI_FAILURE);
686
687 case DDI_SUSPEND:
688 return (ixgbe_suspend(devinfo));
689
690 case DDI_DETACH:
691 break;
692 }
693
694 /*
695 * Get the pointer to the driver private data structure
696 */
697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
698 if (ixgbe == NULL)
699 return (DDI_FAILURE);
700
701 /*
702 * If the device is still running, it needs to be stopped first.
703 * This check is necessary because under some specific circumstances,
704 * the detach routine can be called without stopping the interface
705 * first.
706 */
707 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
709 mutex_enter(&ixgbe->gen_lock);
710 ixgbe_stop(ixgbe, B_TRUE);
711 mutex_exit(&ixgbe->gen_lock);
712 /* Disable and stop the watchdog timer */
713 ixgbe_disable_watchdog_timer(ixgbe);
714 }
715
716 /*
717 * Check if there are still rx buffers held by the upper layer.
718 * If so, fail the detach.
719 */
720 if (!ixgbe_rx_drain(ixgbe))
721 return (DDI_FAILURE);
722
723 /*
724 * Do the remaining unconfigure routines
725 */
726 ixgbe_unconfigure(devinfo, ixgbe);
727
728 return (DDI_SUCCESS);
729 }
730
731 /*
732 * quiesce(9E) entry point.
733 *
734 * This function is called when the system is single-threaded at high
735 * PIL with preemption disabled. Therefore, this function must not be
736 * blocked.
737 *
738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
739 * DDI_FAILURE indicates an error condition and should almost never happen.
740 */
741 static int
742 ixgbe_quiesce(dev_info_t *devinfo)
743 {
744 ixgbe_t *ixgbe;
745 struct ixgbe_hw *hw;
746
747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
748
749 if (ixgbe == NULL)
750 return (DDI_FAILURE);
751
752 hw = &ixgbe->hw;
753
754 /*
755 * Disable the adapter interrupts
756 */
757 ixgbe_disable_adapter_interrupts(ixgbe);
758
759 /*
760 * Tell firmware driver is no longer in control
761 */
762 ixgbe_release_driver_control(hw);
763
764 /*
765 * Reset the chipset
766 */
767 (void) ixgbe_reset_hw(hw);
768
769 /*
770 * Reset PHY
771 */
772 (void) ixgbe_reset_phy(hw);
773
774 return (DDI_SUCCESS);
775 }
776
777 static void
778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
779 {
780 /*
781 * Disable interrupt
782 */
783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
784 (void) ixgbe_disable_intrs(ixgbe);
785 }
786
787 /*
788 * remove the link check timer
789 */
790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
791 if (ixgbe->periodic_id != NULL) {
792 ddi_periodic_delete(ixgbe->periodic_id);
793 ixgbe->periodic_id = NULL;
794 }
795 }
796
797 /*
798 * Unregister MAC
799 */
800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
801 (void) mac_unregister(ixgbe->mac_hdl);
802 }
803
804 /*
805 * Free statistics
806 */
807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
809 }
810
811 /*
812 * Remove interrupt handlers
813 */
814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
815 ixgbe_rem_intr_handlers(ixgbe);
816 }
817
818 /*
819 * Remove taskq for sfp-status-change
820 */
821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
822 ddi_taskq_destroy(ixgbe->sfp_taskq);
823 }
824
825 /*
826 * Remove taskq for over-temp
827 */
828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
829 ddi_taskq_destroy(ixgbe->overtemp_taskq);
830 }
831
832 /*
833 * Remove taskq for external PHYs
834 */
835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
836 ddi_taskq_destroy(ixgbe->phy_taskq);
837 }
838
839 /*
840 * Remove interrupts
841 */
842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
843 ixgbe_rem_intrs(ixgbe);
844 }
845
846 /*
847 * Unregister interrupt callback handler
848 */
849 (void) ddi_cb_unregister(ixgbe->cb_hdl);
850
851 /*
852 * Remove driver properties
853 */
854 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
855 (void) ddi_prop_remove_all(devinfo);
856 }
857
858 /*
859 * Stop the chipset
860 */
861 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
862 mutex_enter(&ixgbe->gen_lock);
863 ixgbe_chip_stop(ixgbe);
864 mutex_exit(&ixgbe->gen_lock);
865 }
866
867 /*
868 * Free register handle
869 */
870 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
871 if (ixgbe->osdep.reg_handle != NULL)
872 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
873 }
874
875 /*
876 * Free PCI config handle
877 */
878 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
879 if (ixgbe->osdep.cfg_handle != NULL)
880 pci_config_teardown(&ixgbe->osdep.cfg_handle);
881 }
882
883 /*
884 * Free locks
885 */
886 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
887 ixgbe_destroy_locks(ixgbe);
888 }
889
890 /*
891 * Free the rx/tx rings
892 */
893 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
894 ixgbe_free_rings(ixgbe);
895 }
896
897 /*
898 * Unregister FMA capabilities
899 */
900 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
901 ixgbe_fm_fini(ixgbe);
902 }
903
904 /*
905 * Free the driver data structure
906 */
907 kmem_free(ixgbe, sizeof (ixgbe_t));
908
909 ddi_set_driver_private(devinfo, NULL);
910 }
911
912 /*
913 * ixgbe_register_mac - Register the driver and its function pointers with
914 * the GLD interface.
915 */
916 static int
917 ixgbe_register_mac(ixgbe_t *ixgbe)
918 {
919 struct ixgbe_hw *hw = &ixgbe->hw;
920 mac_register_t *mac;
921 int status;
922
923 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
924 return (IXGBE_FAILURE);
925
926 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
927 mac->m_driver = ixgbe;
928 mac->m_dip = ixgbe->dip;
929 mac->m_src_addr = hw->mac.addr;
930 mac->m_callbacks = &ixgbe_m_callbacks;
931 mac->m_min_sdu = 0;
932 mac->m_max_sdu = ixgbe->default_mtu;
933 mac->m_margin = VLAN_TAGSZ;
934 mac->m_priv_props = ixgbe_priv_props;
935 mac->m_v12n = MAC_VIRT_LEVEL1;
936
937 status = mac_register(mac, &ixgbe->mac_hdl);
938
939 mac_free(mac);
940
941 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
942 }
943
944 /*
945 * ixgbe_identify_hardware - Identify the type of the chipset.
946 */
947 static int
948 ixgbe_identify_hardware(ixgbe_t *ixgbe)
949 {
950 struct ixgbe_hw *hw = &ixgbe->hw;
951 struct ixgbe_osdep *osdep = &ixgbe->osdep;
952
953 /*
954 * Get the device id
955 */
956 hw->vendor_id =
957 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
958 hw->device_id =
959 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
960 hw->revision_id =
961 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
962 hw->subsystem_device_id =
963 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
964 hw->subsystem_vendor_id =
965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
966
967 /*
968 * Set the mac type of the adapter based on the device id
969 */
970 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
971 return (IXGBE_FAILURE);
972 }
973
974 /*
975 * Install adapter capabilities
976 */
977 switch (hw->mac.type) {
978 case ixgbe_mac_82598EB:
979 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
980 ixgbe->capab = &ixgbe_82598eb_cap;
981
982 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
983 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
984 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
985 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
986 }
987 break;
988
989 case ixgbe_mac_82599EB:
990 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
991 ixgbe->capab = &ixgbe_82599eb_cap;
992
993 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
994 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
995 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
996 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
997 }
998 break;
999
1000 case ixgbe_mac_X540:
1001 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1002 ixgbe->capab = &ixgbe_X540_cap;
1003 /*
1004 * For now, X540 is all set in its capab structure.
1005 * As other X540 variants show up, things can change here.
1006 */
1007 break;
1008
1009 case ixgbe_mac_X550:
1010 case ixgbe_mac_X550EM_x:
1011 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1012 ixgbe->capab = &ixgbe_X550_cap;
1013
1014 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
1015 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1016
1017 /*
1018 * Link detection on X552 SFP+ and X552/X557-AT
1019 */
1020 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1021 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1022 ixgbe->capab->other_intr |=
1023 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1024 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1025 }
1026 break;
1027
1028 default:
1029 IXGBE_DEBUGLOG_1(ixgbe,
1030 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1031 hw->mac.type);
1032 return (IXGBE_FAILURE);
1033 }
1034
1035 return (IXGBE_SUCCESS);
1036 }
1037
1038 /*
1039 * ixgbe_regs_map - Map the device registers.
1040 *
1041 */
1042 static int
1043 ixgbe_regs_map(ixgbe_t *ixgbe)
1044 {
1045 dev_info_t *devinfo = ixgbe->dip;
1046 struct ixgbe_hw *hw = &ixgbe->hw;
1047 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1048 off_t mem_size;
1049
1050 /*
1051 * First get the size of device registers to be mapped.
1052 */
1053 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1054 != DDI_SUCCESS) {
1055 return (IXGBE_FAILURE);
1056 }
1057
1058 /*
1059 * Call ddi_regs_map_setup() to map registers
1060 */
1061 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1062 (caddr_t *)&hw->hw_addr, 0,
1063 mem_size, &ixgbe_regs_acc_attr,
1064 &osdep->reg_handle)) != DDI_SUCCESS) {
1065 return (IXGBE_FAILURE);
1066 }
1067
1068 return (IXGBE_SUCCESS);
1069 }
1070
1071 /*
1072 * ixgbe_init_properties - Initialize driver properties.
1073 */
1074 static void
1075 ixgbe_init_properties(ixgbe_t *ixgbe)
1076 {
1077 /*
1078 * Get conf file properties, including link settings
1079 * jumbo frames, ring number, descriptor number, etc.
1080 */
1081 ixgbe_get_conf(ixgbe);
1082 }
1083
1084 /*
1085 * ixgbe_init_driver_settings - Initialize driver settings.
1086 *
1087 * The settings include hardware function pointers, bus information,
1088 * rx/tx rings settings, link state, and any other parameters that
1089 * need to be setup during driver initialization.
1090 */
1091 static int
1092 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1093 {
1094 struct ixgbe_hw *hw = &ixgbe->hw;
1095 dev_info_t *devinfo = ixgbe->dip;
1096 ixgbe_rx_ring_t *rx_ring;
1097 ixgbe_rx_group_t *rx_group;
1098 ixgbe_tx_ring_t *tx_ring;
1099 uint32_t rx_size;
1100 uint32_t tx_size;
1101 uint32_t ring_per_group;
1102 int i;
1103
1104 /*
1105 * Initialize chipset specific hardware function pointers
1106 */
1107 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1108 return (IXGBE_FAILURE);
1109 }
1110
1111 /*
1112 * Get the system page size
1113 */
1114 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1115
1116 /*
1117 * Set rx buffer size
1118 *
1119 * The IP header alignment room is counted in the calculation.
1120 * The rx buffer size is in unit of 1K that is required by the
1121 * chipset hardware.
1122 */
1123 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1124 ixgbe->rx_buf_size = ((rx_size >> 10) +
1125 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1126
1127 /*
1128 * Set tx buffer size
1129 */
1130 tx_size = ixgbe->max_frame_size;
1131 ixgbe->tx_buf_size = ((tx_size >> 10) +
1132 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1133
1134 /*
1135 * Initialize rx/tx rings/groups parameters
1136 */
1137 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1138 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1139 rx_ring = &ixgbe->rx_rings[i];
1140 rx_ring->index = i;
1141 rx_ring->ixgbe = ixgbe;
1142 rx_ring->group_index = i / ring_per_group;
1143 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1144 }
1145
1146 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1147 rx_group = &ixgbe->rx_groups[i];
1148 rx_group->index = i;
1149 rx_group->ixgbe = ixgbe;
1150 }
1151
1152 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1153 tx_ring = &ixgbe->tx_rings[i];
1154 tx_ring->index = i;
1155 tx_ring->ixgbe = ixgbe;
1156 if (ixgbe->tx_head_wb_enable)
1157 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1158 else
1159 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1160
1161 tx_ring->ring_size = ixgbe->tx_ring_size;
1162 tx_ring->free_list_size = ixgbe->tx_ring_size +
1163 (ixgbe->tx_ring_size >> 1);
1164 }
1165
1166 /*
1167 * Initialize values of interrupt throttling rate
1168 */
1169 for (i = 1; i < MAX_INTR_VECTOR; i++)
1170 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1171
1172 /*
1173 * The initial link state should be "unknown"
1174 */
1175 ixgbe->link_state = LINK_STATE_UNKNOWN;
1176
1177 return (IXGBE_SUCCESS);
1178 }
1179
1180 /*
1181 * ixgbe_init_locks - Initialize locks.
1182 */
1183 static void
1184 ixgbe_init_locks(ixgbe_t *ixgbe)
1185 {
1186 ixgbe_rx_ring_t *rx_ring;
1187 ixgbe_tx_ring_t *tx_ring;
1188 int i;
1189
1190 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1191 rx_ring = &ixgbe->rx_rings[i];
1192 mutex_init(&rx_ring->rx_lock, NULL,
1193 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1194 }
1195
1196 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1197 tx_ring = &ixgbe->tx_rings[i];
1198 mutex_init(&tx_ring->tx_lock, NULL,
1199 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1200 mutex_init(&tx_ring->recycle_lock, NULL,
1201 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1202 mutex_init(&tx_ring->tcb_head_lock, NULL,
1203 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1204 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1206 }
1207
1208 mutex_init(&ixgbe->gen_lock, NULL,
1209 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1210
1211 mutex_init(&ixgbe->watchdog_lock, NULL,
1212 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1213 }
1214
1215 /*
1216 * ixgbe_destroy_locks - Destroy locks.
1217 */
1218 static void
1219 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1220 {
1221 ixgbe_rx_ring_t *rx_ring;
1222 ixgbe_tx_ring_t *tx_ring;
1223 int i;
1224
1225 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1226 rx_ring = &ixgbe->rx_rings[i];
1227 mutex_destroy(&rx_ring->rx_lock);
1228 }
1229
1230 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1231 tx_ring = &ixgbe->tx_rings[i];
1232 mutex_destroy(&tx_ring->tx_lock);
1233 mutex_destroy(&tx_ring->recycle_lock);
1234 mutex_destroy(&tx_ring->tcb_head_lock);
1235 mutex_destroy(&tx_ring->tcb_tail_lock);
1236 }
1237
1238 mutex_destroy(&ixgbe->gen_lock);
1239 mutex_destroy(&ixgbe->watchdog_lock);
1240 }
1241
1242 static int
1243 ixgbe_resume(dev_info_t *devinfo)
1244 {
1245 ixgbe_t *ixgbe;
1246 int i;
1247
1248 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1249 if (ixgbe == NULL)
1250 return (DDI_FAILURE);
1251
1252 mutex_enter(&ixgbe->gen_lock);
1253
1254 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1255 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1256 mutex_exit(&ixgbe->gen_lock);
1257 return (DDI_FAILURE);
1258 }
1259
1260 /*
1261 * Enable and start the watchdog timer
1262 */
1263 ixgbe_enable_watchdog_timer(ixgbe);
1264 }
1265
1266 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1267
1268 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1269 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1270 mac_tx_ring_update(ixgbe->mac_hdl,
1271 ixgbe->tx_rings[i].ring_handle);
1272 }
1273 }
1274
1275 mutex_exit(&ixgbe->gen_lock);
1276
1277 return (DDI_SUCCESS);
1278 }
1279
1280 static int
1281 ixgbe_suspend(dev_info_t *devinfo)
1282 {
1283 ixgbe_t *ixgbe;
1284
1285 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1286 if (ixgbe == NULL)
1287 return (DDI_FAILURE);
1288
1289 mutex_enter(&ixgbe->gen_lock);
1290
1291 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1292 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1293 mutex_exit(&ixgbe->gen_lock);
1294 return (DDI_SUCCESS);
1295 }
1296 ixgbe_stop(ixgbe, B_FALSE);
1297
1298 mutex_exit(&ixgbe->gen_lock);
1299
1300 /*
1301 * Disable and stop the watchdog timer
1302 */
1303 ixgbe_disable_watchdog_timer(ixgbe);
1304
1305 return (DDI_SUCCESS);
1306 }
1307
1308 /*
1309 * ixgbe_init - Initialize the device.
1310 */
1311 static int
1312 ixgbe_init(ixgbe_t *ixgbe)
1313 {
1314 struct ixgbe_hw *hw = &ixgbe->hw;
1315 u8 pbanum[IXGBE_PBANUM_LENGTH];
1316 int rv;
1317
1318 mutex_enter(&ixgbe->gen_lock);
1319
1320 /*
1321 * Configure/Initialize hardware
1322 */
1323 rv = ixgbe_init_hw(hw);
1324 if (rv != IXGBE_SUCCESS) {
1325 switch (rv) {
1326
1327 /*
1328 * The first three errors are not prohibitive to us progressing
1329 * further, and are maily advisory in nature. In the case of a
1330 * SFP module not being present or not deemed supported by the
1331 * common code, we adivse the operator of this fact but carry on
1332 * instead of failing hard, as SFPs can be inserted or replaced
1333 * while the driver is running. In the case of a unknown error,
1334 * we fail-hard, logging the reason and emitting a FMA event.
1335 */
1336 case IXGBE_ERR_EEPROM_VERSION:
1337 ixgbe_error(ixgbe,
1338 "This Intel 10Gb Ethernet device is pre-release and"
1339 " contains outdated firmware. Please contact your"
1340 " hardware vendor for a replacement.");
1341 break;
1342 case IXGBE_ERR_SFP_NOT_PRESENT:
1343 ixgbe_error(ixgbe,
1344 "No SFP+ module detected on this interface. Please "
1345 "install a supported SFP+ module for this "
1346 "interface to become operational.");
1347 break;
1348 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1349 ixgbe_error(ixgbe,
1350 "Unsupported SFP+ module detected. Please replace "
1351 "it with a supported SFP+ module per Intel "
1352 "documentation, or bypass this check with "
1353 "allow_unsupported_sfp=1 in ixgbe.conf.");
1354 break;
1355 default:
1356 ixgbe_error(ixgbe,
1357 "Failed to initialize hardware. ixgbe_init_hw "
1358 "returned %d", rv);
1359 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1360 goto init_fail;
1361 }
1362 }
1363
1364 /*
1365 * Need to init eeprom before validating the checksum.
1366 */
1367 if (ixgbe_init_eeprom_params(hw) < 0) {
1368 ixgbe_error(ixgbe,
1369 "Unable to intitialize the eeprom interface.");
1370 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1371 goto init_fail;
1372 }
1373
1374 /*
1375 * NVM validation
1376 */
1377 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1378 /*
1379 * Some PCI-E parts fail the first check due to
1380 * the link being in sleep state. Call it again,
1381 * if it fails a second time it's a real issue.
1382 */
1383 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1384 ixgbe_error(ixgbe,
1385 "Invalid NVM checksum. Please contact "
1386 "the vendor to update the NVM.");
1387 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1388 goto init_fail;
1389 }
1390 }
1391
1392 /*
1393 * Setup default flow control thresholds - enable/disable
1394 * & flow control type is controlled by ixgbe.conf
1395 */
1396 hw->fc.high_water[0] = DEFAULT_FCRTH;
1397 hw->fc.low_water[0] = DEFAULT_FCRTL;
1398 hw->fc.pause_time = DEFAULT_FCPAUSE;
1399 hw->fc.send_xon = B_TRUE;
1400
1401 /*
1402 * Initialize flow control
1403 */
1404 (void) ixgbe_start_hw(hw);
1405
1406 /*
1407 * Initialize link settings
1408 */
1409 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1410
1411 /*
1412 * Initialize the chipset hardware
1413 */
1414 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1415 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1416 goto init_fail;
1417 }
1418
1419 /*
1420 * Read identifying information and place in devinfo.
1421 */
1422 pbanum[0] = '\0';
1423 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1424 if (*pbanum != '\0') {
1425 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1426 "printed-board-assembly", (char *)pbanum);
1427 }
1428
1429 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1430 goto init_fail;
1431 }
1432
1433 mutex_exit(&ixgbe->gen_lock);
1434 return (IXGBE_SUCCESS);
1435
1436 init_fail:
1437 /*
1438 * Reset PHY
1439 */
1440 (void) ixgbe_reset_phy(hw);
1441
1442 mutex_exit(&ixgbe->gen_lock);
1443 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1444 return (IXGBE_FAILURE);
1445 }
1446
1447 /*
1448 * ixgbe_chip_start - Initialize and start the chipset hardware.
1449 */
1450 static int
1451 ixgbe_chip_start(ixgbe_t *ixgbe)
1452 {
1453 struct ixgbe_hw *hw = &ixgbe->hw;
1454 int i;
1455
1456 ASSERT(mutex_owned(&ixgbe->gen_lock));
1457
1458 /*
1459 * Get the mac address
1460 * This function should handle SPARC case correctly.
1461 */
1462 if (!ixgbe_find_mac_address(ixgbe)) {
1463 ixgbe_error(ixgbe, "Failed to get the mac address");
1464 return (IXGBE_FAILURE);
1465 }
1466
1467 /*
1468 * Validate the mac address
1469 */
1470 (void) ixgbe_init_rx_addrs(hw);
1471 if (!is_valid_mac_addr(hw->mac.addr)) {
1472 ixgbe_error(ixgbe, "Invalid mac address");
1473 return (IXGBE_FAILURE);
1474 }
1475
1476 /*
1477 * Re-enable relaxed ordering for performance. It is disabled
1478 * by default in the hardware init.
1479 */
1480 if (ixgbe->relax_order_enable == B_TRUE)
1481 ixgbe_enable_relaxed_ordering(hw);
1482
1483 /*
1484 * Setup adapter interrupt vectors
1485 */
1486 ixgbe_setup_adapter_vector(ixgbe);
1487
1488 /*
1489 * Initialize unicast addresses.
1490 */
1491 ixgbe_init_unicst(ixgbe);
1492
1493 /*
1494 * Setup and initialize the mctable structures.
1495 */
1496 ixgbe_setup_multicst(ixgbe);
1497
1498 /*
1499 * Set interrupt throttling rate
1500 */
1501 for (i = 0; i < ixgbe->intr_cnt; i++) {
1502 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1503 }
1504
1505 /*
1506 * Disable Wake-on-LAN
1507 */
1508 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1509
1510 /*
1511 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1512 * Due to issues with EEE in e1000g/igb, we disable this by default
1513 * as a precautionary measure.
1514 *
1515 * Currently, the only known adapter which supports EEE in the ixgbe
1516 * line is 8086,15AB (IXGBE_DEV_ID_X550EM_X_KR), and only after the
1517 * first revision of it, as well as any X550 with MAC type 6 (non-EM)
1518 */
1519 (void) ixgbe_setup_eee(hw, B_FALSE);
1520
1521 /*
1522 * Turn on any present SFP Tx laser
1523 */
1524 ixgbe_enable_tx_laser(hw);
1525
1526 /*
1527 * Power on the PHY
1528 */
1529 (void) ixgbe_set_phy_power(hw, B_TRUE);
1530
1531 /*
1532 * Save the state of the PHY
1533 */
1534 ixgbe_get_hw_state(ixgbe);
1535
1536 /*
1537 * Make sure driver has control
1538 */
1539 ixgbe_get_driver_control(hw);
1540
1541 return (IXGBE_SUCCESS);
1542 }
1543
1544 /*
1545 * ixgbe_chip_stop - Stop the chipset hardware
1546 */
1547 static void
1548 ixgbe_chip_stop(ixgbe_t *ixgbe)
1549 {
1550 struct ixgbe_hw *hw = &ixgbe->hw;
1551 int rv;
1552
1553 ASSERT(mutex_owned(&ixgbe->gen_lock));
1554
1555 /*
1556 * Stop interupt generation and disable Tx unit
1557 */
1558 hw->adapter_stopped = B_FALSE;
1559 (void) ixgbe_stop_adapter(hw);
1560
1561 /*
1562 * Reset the chipset
1563 */
1564 (void) ixgbe_reset_hw(hw);
1565
1566 /*
1567 * Reset PHY
1568 */
1569 (void) ixgbe_reset_phy(hw);
1570
1571 /*
1572 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1573 * the PHY while doing so. Else, just power down the PHY.
1574 */
1575 if (hw->phy.ops.enter_lplu != NULL) {
1576 hw->phy.reset_disable = B_TRUE;
1577 rv = hw->phy.ops.enter_lplu(hw);
1578 if (rv != IXGBE_SUCCESS)
1579 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1580 hw->phy.reset_disable = B_FALSE;
1581 } else {
1582 (void) ixgbe_set_phy_power(hw, B_FALSE);
1583 }
1584
1585 /*
1586 * Turn off any present SFP Tx laser
1587 * Expected for health and safety reasons
1588 */
1589 ixgbe_disable_tx_laser(hw);
1590
1591 /*
1592 * Tell firmware driver is no longer in control
1593 */
1594 ixgbe_release_driver_control(hw);
1595
1596 }
1597
1598 /*
1599 * ixgbe_reset - Reset the chipset and re-start the driver.
1600 *
1601 * It involves stopping and re-starting the chipset,
1602 * and re-configuring the rx/tx rings.
1603 */
1604 static int
1605 ixgbe_reset(ixgbe_t *ixgbe)
1606 {
1607 int i;
1608
1609 /*
1610 * Disable and stop the watchdog timer
1611 */
1612 ixgbe_disable_watchdog_timer(ixgbe);
1613
1614 mutex_enter(&ixgbe->gen_lock);
1615
1616 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1617 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1618
1619 ixgbe_stop(ixgbe, B_FALSE);
1620
1621 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1622 mutex_exit(&ixgbe->gen_lock);
1623 return (IXGBE_FAILURE);
1624 }
1625
1626 /*
1627 * After resetting, need to recheck the link status.
1628 */
1629 ixgbe->link_check_complete = B_FALSE;
1630 ixgbe->link_check_hrtime = gethrtime() +
1631 (IXGBE_LINK_UP_TIME * 100000000ULL);
1632
1633 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1634
1635 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1636 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1637 mac_tx_ring_update(ixgbe->mac_hdl,
1638 ixgbe->tx_rings[i].ring_handle);
1639 }
1640 }
1641
1642 mutex_exit(&ixgbe->gen_lock);
1643
1644 /*
1645 * Enable and start the watchdog timer
1646 */
1647 ixgbe_enable_watchdog_timer(ixgbe);
1648
1649 return (IXGBE_SUCCESS);
1650 }
1651
1652 /*
1653 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1654 */
1655 static void
1656 ixgbe_tx_clean(ixgbe_t *ixgbe)
1657 {
1658 ixgbe_tx_ring_t *tx_ring;
1659 tx_control_block_t *tcb;
1660 link_list_t pending_list;
1661 uint32_t desc_num;
1662 int i, j;
1663
1664 LINK_LIST_INIT(&pending_list);
1665
1666 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1667 tx_ring = &ixgbe->tx_rings[i];
1668
1669 mutex_enter(&tx_ring->recycle_lock);
1670
1671 /*
1672 * Clean the pending tx data - the pending packets in the
1673 * work_list that have no chances to be transmitted again.
1674 *
1675 * We must ensure the chipset is stopped or the link is down
1676 * before cleaning the transmit packets.
1677 */
1678 desc_num = 0;
1679 for (j = 0; j < tx_ring->ring_size; j++) {
1680 tcb = tx_ring->work_list[j];
1681 if (tcb != NULL) {
1682 desc_num += tcb->desc_num;
1683
1684 tx_ring->work_list[j] = NULL;
1685
1686 ixgbe_free_tcb(tcb);
1687
1688 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1689 }
1690 }
1691
1692 if (desc_num > 0) {
1693 atomic_add_32(&tx_ring->tbd_free, desc_num);
1694 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1695
1696 /*
1697 * Reset the head and tail pointers of the tbd ring;
1698 * Reset the writeback head if it's enable.
1699 */
1700 tx_ring->tbd_head = 0;
1701 tx_ring->tbd_tail = 0;
1702 if (ixgbe->tx_head_wb_enable)
1703 *tx_ring->tbd_head_wb = 0;
1704
1705 IXGBE_WRITE_REG(&ixgbe->hw,
1706 IXGBE_TDH(tx_ring->index), 0);
1707 IXGBE_WRITE_REG(&ixgbe->hw,
1708 IXGBE_TDT(tx_ring->index), 0);
1709 }
1710
1711 mutex_exit(&tx_ring->recycle_lock);
1712
1713 /*
1714 * Add the tx control blocks in the pending list to
1715 * the free list.
1716 */
1717 ixgbe_put_free_list(tx_ring, &pending_list);
1718 }
1719 }
1720
1721 /*
1722 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1723 * transmitted.
1724 */
1725 static boolean_t
1726 ixgbe_tx_drain(ixgbe_t *ixgbe)
1727 {
1728 ixgbe_tx_ring_t *tx_ring;
1729 boolean_t done;
1730 int i, j;
1731
1732 /*
1733 * Wait for a specific time to allow pending tx packets
1734 * to be transmitted.
1735 *
1736 * Check the counter tbd_free to see if transmission is done.
1737 * No lock protection is needed here.
1738 *
1739 * Return B_TRUE if all pending packets have been transmitted;
1740 * Otherwise return B_FALSE;
1741 */
1742 for (i = 0; i < TX_DRAIN_TIME; i++) {
1743
1744 done = B_TRUE;
1745 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1746 tx_ring = &ixgbe->tx_rings[j];
1747 done = done &&
1748 (tx_ring->tbd_free == tx_ring->ring_size);
1749 }
1750
1751 if (done)
1752 break;
1753
1754 msec_delay(1);
1755 }
1756
1757 return (done);
1758 }
1759
1760 /*
1761 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1762 */
1763 static boolean_t
1764 ixgbe_rx_drain(ixgbe_t *ixgbe)
1765 {
1766 boolean_t done = B_TRUE;
1767 int i;
1768
1769 /*
1770 * Polling the rx free list to check if those rx buffers held by
1771 * the upper layer are released.
1772 *
1773 * Check the counter rcb_free to see if all pending buffers are
1774 * released. No lock protection is needed here.
1775 *
1776 * Return B_TRUE if all pending buffers have been released;
1777 * Otherwise return B_FALSE;
1778 */
1779 for (i = 0; i < RX_DRAIN_TIME; i++) {
1780 done = (ixgbe->rcb_pending == 0);
1781
1782 if (done)
1783 break;
1784
1785 msec_delay(1);
1786 }
1787
1788 return (done);
1789 }
1790
1791 /*
1792 * ixgbe_start - Start the driver/chipset.
1793 */
1794 int
1795 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1796 {
1797 struct ixgbe_hw *hw = &ixgbe->hw;
1798 int i;
1799
1800 ASSERT(mutex_owned(&ixgbe->gen_lock));
1801
1802 if (alloc_buffer) {
1803 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1804 ixgbe_error(ixgbe,
1805 "Failed to allocate software receive rings");
1806 return (IXGBE_FAILURE);
1807 }
1808
1809 /* Allocate buffers for all the rx/tx rings */
1810 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1811 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1812 return (IXGBE_FAILURE);
1813 }
1814
1815 ixgbe->tx_ring_init = B_TRUE;
1816 } else {
1817 ixgbe->tx_ring_init = B_FALSE;
1818 }
1819
1820 for (i = 0; i < ixgbe->num_rx_rings; i++)
1821 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1822 for (i = 0; i < ixgbe->num_tx_rings; i++)
1823 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1824
1825 /*
1826 * Start the chipset hardware
1827 */
1828 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1829 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1830 goto start_failure;
1831 }
1832
1833 /*
1834 * Configure link now for X550
1835 *
1836 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1837 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1838 * the resting state of the link would be the maximum speed that
1839 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1840 * so we never bothered with explicitly setting the link to 10Gb as it
1841 * would already be at that state on driver attach. With X550, we must
1842 * trigger a re-negotiation of the link in order to switch from a LPLU
1843 * 1Gb link to 10Gb (cable and link partner permitting.)
1844 */
1845 if (hw->mac.type == ixgbe_mac_X550 ||
1846 hw->mac.type == ixgbe_mac_X550EM_x) {
1847 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1848 ixgbe_get_hw_state(ixgbe);
1849 }
1850
1851 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1852 goto start_failure;
1853 }
1854
1855 /*
1856 * Setup the rx/tx rings
1857 */
1858 ixgbe_setup_rings(ixgbe);
1859
1860 /*
1861 * ixgbe_start() will be called when resetting, however if reset
1862 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1863 * before enabling the interrupts.
1864 */
1865 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1866 | IXGBE_STALL| IXGBE_OVERTEMP));
1867
1868 /*
1869 * Enable adapter interrupts
1870 * The interrupts must be enabled after the driver state is START
1871 */
1872 ixgbe_enable_adapter_interrupts(ixgbe);
1873
1874 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1875 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1876 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1877 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1878
1879 return (IXGBE_SUCCESS);
1880
1881 start_failure:
1882 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1883 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1884 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1885 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1886
1887 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1888
1889 return (IXGBE_FAILURE);
1890 }
1891
1892 /*
1893 * ixgbe_stop - Stop the driver/chipset.
1894 */
1895 void
1896 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1897 {
1898 int i;
1899
1900 ASSERT(mutex_owned(&ixgbe->gen_lock));
1901
1902 /*
1903 * Disable the adapter interrupts
1904 */
1905 ixgbe_disable_adapter_interrupts(ixgbe);
1906
1907 /*
1908 * Drain the pending tx packets
1909 */
1910 (void) ixgbe_tx_drain(ixgbe);
1911
1912 for (i = 0; i < ixgbe->num_rx_rings; i++)
1913 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1914 for (i = 0; i < ixgbe->num_tx_rings; i++)
1915 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1916
1917 /*
1918 * Stop the chipset hardware
1919 */
1920 ixgbe_chip_stop(ixgbe);
1921
1922 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1923 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1924 }
1925
1926 /*
1927 * Clean the pending tx data/resources
1928 */
1929 ixgbe_tx_clean(ixgbe);
1930
1931 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1932 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1933 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1934 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1935
1936 if (ixgbe->link_state == LINK_STATE_UP) {
1937 ixgbe->link_state = LINK_STATE_UNKNOWN;
1938 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1939 }
1940
1941 if (free_buffer) {
1942 /*
1943 * Release the DMA/memory resources of rx/tx rings
1944 */
1945 ixgbe_free_dma(ixgbe);
1946 ixgbe_free_rx_data(ixgbe);
1947 }
1948 }
1949
1950 /*
1951 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1952 */
1953 /* ARGSUSED */
1954 static int
1955 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1956 void *arg1, void *arg2)
1957 {
1958 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1959
1960 switch (cbaction) {
1961 /* IRM callback */
1962 int count;
1963 case DDI_CB_INTR_ADD:
1964 case DDI_CB_INTR_REMOVE:
1965 count = (int)(uintptr_t)cbarg;
1966 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1967 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1968 int, ixgbe->intr_cnt);
1969 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1970 DDI_SUCCESS) {
1971 ixgbe_error(ixgbe,
1972 "IRM CB: Failed to adjust interrupts");
1973 goto cb_fail;
1974 }
1975 break;
1976 default:
1977 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1978 cbaction);
1979 return (DDI_ENOTSUP);
1980 }
1981 return (DDI_SUCCESS);
1982 cb_fail:
1983 return (DDI_FAILURE);
1984 }
1985
1986 /*
1987 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1988 */
1989 static int
1990 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1991 {
1992 int i, rc, actual;
1993
1994 if (count == 0)
1995 return (DDI_SUCCESS);
1996
1997 if ((cbaction == DDI_CB_INTR_ADD &&
1998 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1999 (cbaction == DDI_CB_INTR_REMOVE &&
2000 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2001 return (DDI_FAILURE);
2002
2003 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2004 return (DDI_FAILURE);
2005 }
2006
2007 for (i = 0; i < ixgbe->num_rx_rings; i++)
2008 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2009 for (i = 0; i < ixgbe->num_tx_rings; i++)
2010 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2011
2012 mutex_enter(&ixgbe->gen_lock);
2013 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2014 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2015 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2016 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2017
2018 ixgbe_stop(ixgbe, B_FALSE);
2019 /*
2020 * Disable interrupts
2021 */
2022 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2023 rc = ixgbe_disable_intrs(ixgbe);
2024 ASSERT(rc == IXGBE_SUCCESS);
2025 }
2026 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2027
2028 /*
2029 * Remove interrupt handlers
2030 */
2031 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2032 ixgbe_rem_intr_handlers(ixgbe);
2033 }
2034 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2035
2036 /*
2037 * Clear vect_map
2038 */
2039 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2040 switch (cbaction) {
2041 case DDI_CB_INTR_ADD:
2042 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2043 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2044 DDI_INTR_ALLOC_NORMAL);
2045 if (rc != DDI_SUCCESS || actual != count) {
2046 ixgbe_log(ixgbe, "Adjust interrupts failed."
2047 "return: %d, irm cb size: %d, actual: %d",
2048 rc, count, actual);
2049 goto intr_adjust_fail;
2050 }
2051 ixgbe->intr_cnt += count;
2052 break;
2053
2054 case DDI_CB_INTR_REMOVE:
2055 for (i = ixgbe->intr_cnt - count;
2056 i < ixgbe->intr_cnt; i ++) {
2057 rc = ddi_intr_free(ixgbe->htable[i]);
2058 ixgbe->htable[i] = NULL;
2059 if (rc != DDI_SUCCESS) {
2060 ixgbe_log(ixgbe, "Adjust interrupts failed."
2061 "return: %d, irm cb size: %d, actual: %d",
2062 rc, count, actual);
2063 goto intr_adjust_fail;
2064 }
2065 }
2066 ixgbe->intr_cnt -= count;
2067 break;
2068 }
2069
2070 /*
2071 * Get priority for first vector, assume remaining are all the same
2072 */
2073 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2074 if (rc != DDI_SUCCESS) {
2075 ixgbe_log(ixgbe,
2076 "Get interrupt priority failed: %d", rc);
2077 goto intr_adjust_fail;
2078 }
2079 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2080 if (rc != DDI_SUCCESS) {
2081 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2082 goto intr_adjust_fail;
2083 }
2084 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2085
2086 /*
2087 * Map rings to interrupt vectors
2088 */
2089 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2090 ixgbe_error(ixgbe,
2091 "IRM CB: Failed to map interrupts to vectors");
2092 goto intr_adjust_fail;
2093 }
2094
2095 /*
2096 * Add interrupt handlers
2097 */
2098 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2099 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2100 goto intr_adjust_fail;
2101 }
2102 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2103
2104 /*
2105 * Now that mutex locks are initialized, and the chip is also
2106 * initialized, enable interrupts.
2107 */
2108 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2109 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2110 goto intr_adjust_fail;
2111 }
2112 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2113 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2114 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2115 goto intr_adjust_fail;
2116 }
2117 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2118 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2119 ixgbe->ixgbe_state |= IXGBE_STARTED;
2120 mutex_exit(&ixgbe->gen_lock);
2121
2122 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2123 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2124 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2125 }
2126 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2127 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2128 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2129 }
2130
2131 /* Wakeup all Tx rings */
2132 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2133 mac_tx_ring_update(ixgbe->mac_hdl,
2134 ixgbe->tx_rings[i].ring_handle);
2135 }
2136
2137 IXGBE_DEBUGLOG_3(ixgbe,
2138 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2139 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2140 return (DDI_SUCCESS);
2141
2142 intr_adjust_fail:
2143 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2144 mutex_exit(&ixgbe->gen_lock);
2145 return (DDI_FAILURE);
2146 }
2147
2148 /*
2149 * ixgbe_intr_cb_register - Register interrupt callback function.
2150 */
2151 static int
2152 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2153 {
2154 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2155 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2156 return (IXGBE_FAILURE);
2157 }
2158 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2159 return (IXGBE_SUCCESS);
2160 }
2161
2162 /*
2163 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2164 */
2165 static int
2166 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2167 {
2168 /*
2169 * Allocate memory space for rx rings
2170 */
2171 ixgbe->rx_rings = kmem_zalloc(
2172 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2173 KM_NOSLEEP);
2174
2175 if (ixgbe->rx_rings == NULL) {
2176 return (IXGBE_FAILURE);
2177 }
2178
2179 /*
2180 * Allocate memory space for tx rings
2181 */
2182 ixgbe->tx_rings = kmem_zalloc(
2183 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2184 KM_NOSLEEP);
2185
2186 if (ixgbe->tx_rings == NULL) {
2187 kmem_free(ixgbe->rx_rings,
2188 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2189 ixgbe->rx_rings = NULL;
2190 return (IXGBE_FAILURE);
2191 }
2192
2193 /*
2194 * Allocate memory space for rx ring groups
2195 */
2196 ixgbe->rx_groups = kmem_zalloc(
2197 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2198 KM_NOSLEEP);
2199
2200 if (ixgbe->rx_groups == NULL) {
2201 kmem_free(ixgbe->rx_rings,
2202 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2203 kmem_free(ixgbe->tx_rings,
2204 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2205 ixgbe->rx_rings = NULL;
2206 ixgbe->tx_rings = NULL;
2207 return (IXGBE_FAILURE);
2208 }
2209
2210 return (IXGBE_SUCCESS);
2211 }
2212
2213 /*
2214 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2215 */
2216 static void
2217 ixgbe_free_rings(ixgbe_t *ixgbe)
2218 {
2219 if (ixgbe->rx_rings != NULL) {
2220 kmem_free(ixgbe->rx_rings,
2221 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2222 ixgbe->rx_rings = NULL;
2223 }
2224
2225 if (ixgbe->tx_rings != NULL) {
2226 kmem_free(ixgbe->tx_rings,
2227 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2228 ixgbe->tx_rings = NULL;
2229 }
2230
2231 if (ixgbe->rx_groups != NULL) {
2232 kmem_free(ixgbe->rx_groups,
2233 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2234 ixgbe->rx_groups = NULL;
2235 }
2236 }
2237
2238 static int
2239 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2240 {
2241 ixgbe_rx_ring_t *rx_ring;
2242 int i;
2243
2244 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2245 rx_ring = &ixgbe->rx_rings[i];
2246 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2247 goto alloc_rx_rings_failure;
2248 }
2249 return (IXGBE_SUCCESS);
2250
2251 alloc_rx_rings_failure:
2252 ixgbe_free_rx_data(ixgbe);
2253 return (IXGBE_FAILURE);
2254 }
2255
2256 static void
2257 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2258 {
2259 ixgbe_rx_ring_t *rx_ring;
2260 ixgbe_rx_data_t *rx_data;
2261 int i;
2262
2263 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2264 rx_ring = &ixgbe->rx_rings[i];
2265
2266 mutex_enter(&ixgbe->rx_pending_lock);
2267 rx_data = rx_ring->rx_data;
2268
2269 if (rx_data != NULL) {
2270 rx_data->flag |= IXGBE_RX_STOPPED;
2271
2272 if (rx_data->rcb_pending == 0) {
2273 ixgbe_free_rx_ring_data(rx_data);
2274 rx_ring->rx_data = NULL;
2275 }
2276 }
2277
2278 mutex_exit(&ixgbe->rx_pending_lock);
2279 }
2280 }
2281
2282 /*
2283 * ixgbe_setup_rings - Setup rx/tx rings.
2284 */
2285 static void
2286 ixgbe_setup_rings(ixgbe_t *ixgbe)
2287 {
2288 /*
2289 * Setup the rx/tx rings, including the following:
2290 *
2291 * 1. Setup the descriptor ring and the control block buffers;
2292 * 2. Initialize necessary registers for receive/transmit;
2293 * 3. Initialize software pointers/parameters for receive/transmit;
2294 */
2295 ixgbe_setup_rx(ixgbe);
2296
2297 ixgbe_setup_tx(ixgbe);
2298 }
2299
2300 static void
2301 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2302 {
2303 ixgbe_t *ixgbe = rx_ring->ixgbe;
2304 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2305 struct ixgbe_hw *hw = &ixgbe->hw;
2306 rx_control_block_t *rcb;
2307 union ixgbe_adv_rx_desc *rbd;
2308 uint32_t size;
2309 uint32_t buf_low;
2310 uint32_t buf_high;
2311 uint32_t reg_val;
2312 int i;
2313
2314 ASSERT(mutex_owned(&rx_ring->rx_lock));
2315 ASSERT(mutex_owned(&ixgbe->gen_lock));
2316
2317 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2318 rcb = rx_data->work_list[i];
2319 rbd = &rx_data->rbd_ring[i];
2320
2321 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2322 rbd->read.hdr_addr = NULL;
2323 }
2324
2325 /*
2326 * Initialize the length register
2327 */
2328 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2329 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2330
2331 /*
2332 * Initialize the base address registers
2333 */
2334 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2335 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2336 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2337 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2338
2339 /*
2340 * Setup head & tail pointers
2341 */
2342 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2343 rx_data->ring_size - 1);
2344 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2345
2346 rx_data->rbd_next = 0;
2347 rx_data->lro_first = 0;
2348
2349 /*
2350 * Setup the Receive Descriptor Control Register (RXDCTL)
2351 * PTHRESH=32 descriptors (half the internal cache)
2352 * HTHRESH=0 descriptors (to minimize latency on fetch)
2353 * WTHRESH defaults to 1 (writeback each descriptor)
2354 */
2355 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2356 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2357
2358 /* Not a valid value for 82599, X540 or X550 */
2359 if (hw->mac.type == ixgbe_mac_82598EB) {
2360 reg_val |= 0x0020; /* pthresh */
2361 }
2362 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2363
2364 if (hw->mac.type == ixgbe_mac_82599EB ||
2365 hw->mac.type == ixgbe_mac_X540 ||
2366 hw->mac.type == ixgbe_mac_X550 ||
2367 hw->mac.type == ixgbe_mac_X550EM_x) {
2368 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2369 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2370 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2371 }
2372
2373 /*
2374 * Setup the Split and Replication Receive Control Register.
2375 * Set the rx buffer size and the advanced descriptor type.
2376 */
2377 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2378 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2379 reg_val |= IXGBE_SRRCTL_DROP_EN;
2380 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2381 }
2382
2383 static void
2384 ixgbe_setup_rx(ixgbe_t *ixgbe)
2385 {
2386 ixgbe_rx_ring_t *rx_ring;
2387 struct ixgbe_hw *hw = &ixgbe->hw;
2388 uint32_t reg_val;
2389 uint32_t ring_mapping;
2390 uint32_t i, index;
2391 uint32_t psrtype_rss_bit;
2392
2393 /*
2394 * Ensure that Rx is disabled while setting up
2395 * the Rx unit and Rx descriptor ring(s)
2396 */
2397 ixgbe_disable_rx(hw);
2398
2399 /* PSRTYPE must be configured for 82599 */
2400 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2401 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2402 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2403 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2404 reg_val |= IXGBE_PSRTYPE_L2HDR;
2405 reg_val |= 0x80000000;
2406 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2407 } else {
2408 if (ixgbe->num_rx_groups > 32) {
2409 psrtype_rss_bit = 0x20000000;
2410 } else {
2411 psrtype_rss_bit = 0x40000000;
2412 }
2413 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2414 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2415 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2416 reg_val |= IXGBE_PSRTYPE_L2HDR;
2417 reg_val |= psrtype_rss_bit;
2418 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2419 }
2420 }
2421
2422 /*
2423 * Set filter control in FCTRL to determine types of packets are passed
2424 * up to the driver.
2425 * - Pass broadcast packets.
2426 * - Do not pass flow control pause frames (82598-specific)
2427 */
2428 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2429 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2430 if (hw->mac.type == ixgbe_mac_82598EB) {
2431 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2432 }
2433 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2434
2435 /*
2436 * Hardware checksum settings
2437 */
2438 if (ixgbe->rx_hcksum_enable) {
2439 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2440 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2441 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2442 }
2443
2444 /*
2445 * Setup VMDq and RSS for multiple receive queues
2446 */
2447 switch (ixgbe->classify_mode) {
2448 case IXGBE_CLASSIFY_RSS:
2449 /*
2450 * One group, only RSS is needed when more than
2451 * one ring enabled.
2452 */
2453 ixgbe_setup_rss(ixgbe);
2454 break;
2455
2456 case IXGBE_CLASSIFY_VMDQ:
2457 /*
2458 * Multiple groups, each group has one ring,
2459 * only VMDq is needed.
2460 */
2461 ixgbe_setup_vmdq(ixgbe);
2462 break;
2463
2464 case IXGBE_CLASSIFY_VMDQ_RSS:
2465 /*
2466 * Multiple groups and multiple rings, both
2467 * VMDq and RSS are needed.
2468 */
2469 ixgbe_setup_vmdq_rss(ixgbe);
2470 break;
2471
2472 default:
2473 break;
2474 }
2475
2476 /*
2477 * Enable the receive unit. This must be done after filter
2478 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2479 * 82598 is the only adapter which defines this RXCTRL option.
2480 */
2481 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2482 if (hw->mac.type == ixgbe_mac_82598EB)
2483 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2484 reg_val |= IXGBE_RXCTRL_RXEN;
2485 (void) ixgbe_enable_rx_dma(hw, reg_val);
2486
2487 /*
2488 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2489 */
2490 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2491 rx_ring = &ixgbe->rx_rings[i];
2492 ixgbe_setup_rx_ring(rx_ring);
2493 }
2494
2495 /*
2496 * Setup the per-ring statistics mapping.
2497 */
2498 ring_mapping = 0;
2499 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2500 index = ixgbe->rx_rings[i].hw_index;
2501 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2502 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2503 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2504 }
2505
2506 /*
2507 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2508 * by four bytes if the packet has a VLAN field, so includes MTU,
2509 * ethernet header and frame check sequence.
2510 * Register is MAXFRS in 82599.
2511 */
2512 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2513 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2514 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2515 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2516 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2517
2518 /*
2519 * Setup Jumbo Frame enable bit
2520 */
2521 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2522 if (ixgbe->default_mtu > ETHERMTU)
2523 reg_val |= IXGBE_HLREG0_JUMBOEN;
2524 else
2525 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2526 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2527
2528 /*
2529 * Setup RSC for multiple receive queues.
2530 */
2531 if (ixgbe->lro_enable) {
2532 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2533 /*
2534 * Make sure rx_buf_size * MAXDESC not greater
2535 * than 65535.
2536 * Intel recommends 4 for MAXDESC field value.
2537 */
2538 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2539 reg_val |= IXGBE_RSCCTL_RSCEN;
2540 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2541 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2542 else
2543 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2544 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2545 }
2546
2547 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2548 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2549 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2550
2551 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2552 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2553 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2554 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2555
2556 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2557 }
2558 }
2559
2560 static void
2561 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2562 {
2563 ixgbe_t *ixgbe = tx_ring->ixgbe;
2564 struct ixgbe_hw *hw = &ixgbe->hw;
2565 uint32_t size;
2566 uint32_t buf_low;
2567 uint32_t buf_high;
2568 uint32_t reg_val;
2569
2570 ASSERT(mutex_owned(&tx_ring->tx_lock));
2571 ASSERT(mutex_owned(&ixgbe->gen_lock));
2572
2573 /*
2574 * Initialize the length register
2575 */
2576 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2577 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2578
2579 /*
2580 * Initialize the base address registers
2581 */
2582 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2583 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2584 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2585 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2586
2587 /*
2588 * Setup head & tail pointers
2589 */
2590 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2591 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2592
2593 /*
2594 * Setup head write-back
2595 */
2596 if (ixgbe->tx_head_wb_enable) {
2597 /*
2598 * The memory of the head write-back is allocated using
2599 * the extra tbd beyond the tail of the tbd ring.
2600 */
2601 tx_ring->tbd_head_wb = (uint32_t *)
2602 ((uintptr_t)tx_ring->tbd_area.address + size);
2603 *tx_ring->tbd_head_wb = 0;
2604
2605 buf_low = (uint32_t)
2606 (tx_ring->tbd_area.dma_address + size);
2607 buf_high = (uint32_t)
2608 ((tx_ring->tbd_area.dma_address + size) >> 32);
2609
2610 /* Set the head write-back enable bit */
2611 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2612
2613 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2614 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2615
2616 /*
2617 * Turn off relaxed ordering for head write back or it will
2618 * cause problems with the tx recycling
2619 */
2620
2621 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2622 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2623 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2624 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2625 if (hw->mac.type == ixgbe_mac_82598EB) {
2626 IXGBE_WRITE_REG(hw,
2627 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2628 } else {
2629 IXGBE_WRITE_REG(hw,
2630 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2631 }
2632 } else {
2633 tx_ring->tbd_head_wb = NULL;
2634 }
2635
2636 tx_ring->tbd_head = 0;
2637 tx_ring->tbd_tail = 0;
2638 tx_ring->tbd_free = tx_ring->ring_size;
2639
2640 if (ixgbe->tx_ring_init == B_TRUE) {
2641 tx_ring->tcb_head = 0;
2642 tx_ring->tcb_tail = 0;
2643 tx_ring->tcb_free = tx_ring->free_list_size;
2644 }
2645
2646 /*
2647 * Initialize the s/w context structure
2648 */
2649 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2650 }
2651
2652 static void
2653 ixgbe_setup_tx(ixgbe_t *ixgbe)
2654 {
2655 struct ixgbe_hw *hw = &ixgbe->hw;
2656 ixgbe_tx_ring_t *tx_ring;
2657 uint32_t reg_val;
2658 uint32_t ring_mapping;
2659 int i;
2660
2661 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2662 tx_ring = &ixgbe->tx_rings[i];
2663 ixgbe_setup_tx_ring(tx_ring);
2664 }
2665
2666 /*
2667 * Setup the per-ring statistics mapping.
2668 */
2669 ring_mapping = 0;
2670 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2671 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2672 if ((i & 0x3) == 0x3) {
2673 switch (hw->mac.type) {
2674 case ixgbe_mac_82598EB:
2675 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2676 ring_mapping);
2677 break;
2678
2679 case ixgbe_mac_82599EB:
2680 case ixgbe_mac_X540:
2681 case ixgbe_mac_X550:
2682 case ixgbe_mac_X550EM_x:
2683 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2684 ring_mapping);
2685 break;
2686
2687 default:
2688 break;
2689 }
2690
2691 ring_mapping = 0;
2692 }
2693 }
2694 if (i & 0x3) {
2695 switch (hw->mac.type) {
2696 case ixgbe_mac_82598EB:
2697 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2698 break;
2699
2700 case ixgbe_mac_82599EB:
2701 case ixgbe_mac_X540:
2702 case ixgbe_mac_X550:
2703 case ixgbe_mac_X550EM_x:
2704 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2705 break;
2706
2707 default:
2708 break;
2709 }
2710 }
2711
2712 /*
2713 * Enable CRC appending and TX padding (for short tx frames)
2714 */
2715 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2716 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2717 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2718
2719 /*
2720 * enable DMA for 82599, X540 and X550 parts
2721 */
2722 if (hw->mac.type == ixgbe_mac_82599EB ||
2723 hw->mac.type == ixgbe_mac_X540 ||
2724 hw->mac.type == ixgbe_mac_X550 ||
2725 hw->mac.type == ixgbe_mac_X550EM_x) {
2726 /* DMATXCTL.TE must be set after all Tx config is complete */
2727 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2728 reg_val |= IXGBE_DMATXCTL_TE;
2729 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2730
2731 /* Disable arbiter to set MTQC */
2732 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2733 reg_val |= IXGBE_RTTDCS_ARBDIS;
2734 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2735 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2736 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2737 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2738 }
2739
2740 /*
2741 * Enabling tx queues ..
2742 * For 82599 must be done after DMATXCTL.TE is set
2743 */
2744 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2745 tx_ring = &ixgbe->tx_rings[i];
2746 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2747 reg_val |= IXGBE_TXDCTL_ENABLE;
2748 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2749 }
2750 }
2751
2752 /*
2753 * ixgbe_setup_rss - Setup receive-side scaling feature.
2754 */
2755 static void
2756 ixgbe_setup_rss(ixgbe_t *ixgbe)
2757 {
2758 struct ixgbe_hw *hw = &ixgbe->hw;
2759 uint32_t mrqc;
2760
2761 /*
2762 * Initialize RETA/ERETA table
2763 */
2764 ixgbe_setup_rss_table(ixgbe);
2765
2766 /*
2767 * Enable RSS & perform hash on these packet types
2768 */
2769 mrqc = IXGBE_MRQC_RSSEN |
2770 IXGBE_MRQC_RSS_FIELD_IPV4 |
2771 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2772 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2773 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2774 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2775 IXGBE_MRQC_RSS_FIELD_IPV6 |
2776 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2777 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2778 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2779 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2780 }
2781
2782 /*
2783 * ixgbe_setup_vmdq - Setup MAC classification feature
2784 */
2785 static void
2786 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2787 {
2788 struct ixgbe_hw *hw = &ixgbe->hw;
2789 uint32_t vmdctl, i, vtctl;
2790
2791 /*
2792 * Setup the VMDq Control register, enable VMDq based on
2793 * packet destination MAC address:
2794 */
2795 switch (hw->mac.type) {
2796 case ixgbe_mac_82598EB:
2797 /*
2798 * VMDq Enable = 1;
2799 * VMDq Filter = 0; MAC filtering
2800 * Default VMDq output index = 0;
2801 */
2802 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2803 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2804 break;
2805
2806 case ixgbe_mac_82599EB:
2807 case ixgbe_mac_X540:
2808 case ixgbe_mac_X550:
2809 case ixgbe_mac_X550EM_x:
2810 /*
2811 * Enable VMDq-only.
2812 */
2813 vmdctl = IXGBE_MRQC_VMDQEN;
2814 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2815
2816 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2817 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2818 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2819 }
2820
2821 /*
2822 * Enable Virtualization and Replication.
2823 */
2824 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2825 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2826
2827 /*
2828 * Enable receiving packets to all VFs
2829 */
2830 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2831 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2832 break;
2833
2834 default:
2835 break;
2836 }
2837 }
2838
2839 /*
2840 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2841 */
2842 static void
2843 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2844 {
2845 struct ixgbe_hw *hw = &ixgbe->hw;
2846 uint32_t i, mrqc;
2847 uint32_t vtctl, vmdctl;
2848
2849 /*
2850 * Initialize RETA/ERETA table
2851 */
2852 ixgbe_setup_rss_table(ixgbe);
2853
2854 /*
2855 * Enable and setup RSS and VMDq
2856 */
2857 switch (hw->mac.type) {
2858 case ixgbe_mac_82598EB:
2859 /*
2860 * Enable RSS & Setup RSS Hash functions
2861 */
2862 mrqc = IXGBE_MRQC_RSSEN |
2863 IXGBE_MRQC_RSS_FIELD_IPV4 |
2864 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2865 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2866 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2867 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2868 IXGBE_MRQC_RSS_FIELD_IPV6 |
2869 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2870 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2871 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2872 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2873
2874 /*
2875 * Enable and Setup VMDq
2876 * VMDq Filter = 0; MAC filtering
2877 * Default VMDq output index = 0;
2878 */
2879 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2880 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2881 break;
2882
2883 case ixgbe_mac_82599EB:
2884 case ixgbe_mac_X540:
2885 case ixgbe_mac_X550:
2886 case ixgbe_mac_X550EM_x:
2887 /*
2888 * Enable RSS & Setup RSS Hash functions
2889 */
2890 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2891 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2892 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2893 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2894 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2895 IXGBE_MRQC_RSS_FIELD_IPV6 |
2896 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2897 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2898 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2899
2900 /*
2901 * Enable VMDq+RSS.
2902 */
2903 if (ixgbe->num_rx_groups > 32) {
2904 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2905 } else {
2906 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2907 }
2908
2909 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2910
2911 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2912 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2913 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2914 }
2915 break;
2916
2917 default:
2918 break;
2919
2920 }
2921
2922 if (hw->mac.type == ixgbe_mac_82599EB ||
2923 hw->mac.type == ixgbe_mac_X540 ||
2924 hw->mac.type == ixgbe_mac_X550 ||
2925 hw->mac.type == ixgbe_mac_X550EM_x) {
2926 /*
2927 * Enable Virtualization and Replication.
2928 */
2929 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2930 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2931
2932 /*
2933 * Enable receiving packets to all VFs
2934 */
2935 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2936 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2937 }
2938 }
2939
2940 /*
2941 * ixgbe_setup_rss_table - Setup RSS table
2942 */
2943 static void
2944 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
2945 {
2946 struct ixgbe_hw *hw = &ixgbe->hw;
2947 uint32_t i, j;
2948 uint32_t random;
2949 uint32_t reta;
2950 uint32_t ring_per_group;
2951 uint32_t ring;
2952 uint32_t table_size;
2953 uint32_t index_mult;
2954 uint32_t rxcsum;
2955
2956 /*
2957 * Set multiplier for RETA setup and table size based on MAC type.
2958 * RETA table sizes vary by model:
2959 *
2960 * 82598, 82599, X540: 128 table entries.
2961 * X550: 512 table entries.
2962 */
2963 index_mult = 0x1;
2964 table_size = 128;
2965 switch (ixgbe->hw.mac.type) {
2966 case ixgbe_mac_82598EB:
2967 index_mult = 0x11;
2968 break;
2969 case ixgbe_mac_X550:
2970 case ixgbe_mac_X550EM_x:
2971 table_size = 512;
2972 break;
2973 default:
2974 break;
2975 }
2976
2977 /*
2978 * Fill out RSS redirection table. The configuation of the indices is
2979 * hardware-dependent.
2980 *
2981 * 82598: 8 bits wide containing two 4 bit RSS indices
2982 * 82599, X540: 8 bits wide containing one 4 bit RSS index
2983 * X550: 8 bits wide containing one 6 bit RSS index
2984 */
2985 reta = 0;
2986 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2987
2988 for (i = 0, j = 0; i < table_size; i++, j++) {
2989 if (j == ring_per_group) j = 0;
2990
2991 /*
2992 * The low 8 bits are for hash value (n+0);
2993 * The next 8 bits are for hash value (n+1), etc.
2994 */
2995 ring = (j * index_mult);
2996 reta = reta >> 8;
2997 reta = reta | (((uint32_t)ring) << 24);
2998
2999 if ((i & 3) == 3)
3000 /*
3001 * The first 128 table entries are programmed into the
3002 * RETA register, with any beyond that (eg; on X550)
3003 * into ERETA.
3004 */
3005 if (i < 128)
3006 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3007 else
3008 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3009 reta);
3010 reta = 0;
3011 }
3012
3013 /*
3014 * Fill out hash function seeds with a random constant
3015 */
3016 for (i = 0; i < 10; i++) {
3017 (void) random_get_pseudo_bytes((uint8_t *)&random,
3018 sizeof (uint32_t));
3019 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3020 }
3021
3022 /*
3023 * Disable Packet Checksum to enable RSS for multiple receive queues.
3024 * It is an adapter hardware limitation that Packet Checksum is
3025 * mutually exclusive with RSS.
3026 */
3027 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3028 rxcsum |= IXGBE_RXCSUM_PCSD;
3029 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3030 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3031 }
3032
3033 /*
3034 * ixgbe_init_unicst - Initialize the unicast addresses.
3035 */
3036 static void
3037 ixgbe_init_unicst(ixgbe_t *ixgbe)
3038 {
3039 struct ixgbe_hw *hw = &ixgbe->hw;
3040 uint8_t *mac_addr;
3041 int slot;
3042 /*
3043 * Here we should consider two situations:
3044 *
3045 * 1. Chipset is initialized at the first time,
3046 * Clear all the multiple unicast addresses.
3047 *
3048 * 2. Chipset is reset
3049 * Recover the multiple unicast addresses from the
3050 * software data structure to the RAR registers.
3051 */
3052 if (!ixgbe->unicst_init) {
3053 /*
3054 * Initialize the multiple unicast addresses
3055 */
3056 ixgbe->unicst_total = hw->mac.num_rar_entries;
3057 ixgbe->unicst_avail = ixgbe->unicst_total;
3058 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3059 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3060 bzero(mac_addr, ETHERADDRL);
3061 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
3062 ixgbe->unicst_addr[slot].mac.set = 0;
3063 }
3064 ixgbe->unicst_init = B_TRUE;
3065 } else {
3066 /* Re-configure the RAR registers */
3067 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3068 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3069 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3070 (void) ixgbe_set_rar(hw, slot, mac_addr,
3071 ixgbe->unicst_addr[slot].mac.group_index,
3072 IXGBE_RAH_AV);
3073 } else {
3074 bzero(mac_addr, ETHERADDRL);
3075 (void) ixgbe_set_rar(hw, slot, mac_addr,
3076 NULL, NULL);
3077 }
3078 }
3079 }
3080 }
3081
3082 /*
3083 * ixgbe_unicst_find - Find the slot for the specified unicast address
3084 */
3085 int
3086 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3087 {
3088 int slot;
3089
3090 ASSERT(mutex_owned(&ixgbe->gen_lock));
3091
3092 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3093 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3094 mac_addr, ETHERADDRL) == 0)
3095 return (slot);
3096 }
3097
3098 return (-1);
3099 }
3100
3101 /*
3102 * ixgbe_multicst_add - Add a multicst address.
3103 */
3104 int
3105 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3106 {
3107 ASSERT(mutex_owned(&ixgbe->gen_lock));
3108
3109 if ((multiaddr[0] & 01) == 0) {
3110 return (EINVAL);
3111 }
3112
3113 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3114 return (ENOENT);
3115 }
3116
3117 bcopy(multiaddr,
3118 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3119 ixgbe->mcast_count++;
3120
3121 /*
3122 * Update the multicast table in the hardware
3123 */
3124 ixgbe_setup_multicst(ixgbe);
3125
3126 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3127 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3128 return (EIO);
3129 }
3130
3131 return (0);
3132 }
3133
3134 /*
3135 * ixgbe_multicst_remove - Remove a multicst address.
3136 */
3137 int
3138 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3139 {
3140 int i;
3141
3142 ASSERT(mutex_owned(&ixgbe->gen_lock));
3143
3144 for (i = 0; i < ixgbe->mcast_count; i++) {
3145 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3146 ETHERADDRL) == 0) {
3147 for (i++; i < ixgbe->mcast_count; i++) {
3148 ixgbe->mcast_table[i - 1] =
3149 ixgbe->mcast_table[i];
3150 }
3151 ixgbe->mcast_count--;
3152 break;
3153 }
3154 }
3155
3156 /*
3157 * Update the multicast table in the hardware
3158 */
3159 ixgbe_setup_multicst(ixgbe);
3160
3161 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3162 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3163 return (EIO);
3164 }
3165
3166 return (0);
3167 }
3168
3169 /*
3170 * ixgbe_setup_multicast - Setup multicast data structures.
3171 *
3172 * This routine initializes all of the multicast related structures
3173 * and save them in the hardware registers.
3174 */
3175 static void
3176 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3177 {
3178 uint8_t *mc_addr_list;
3179 uint32_t mc_addr_count;
3180 struct ixgbe_hw *hw = &ixgbe->hw;
3181
3182 ASSERT(mutex_owned(&ixgbe->gen_lock));
3183
3184 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3185
3186 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3187 mc_addr_count = ixgbe->mcast_count;
3188
3189 /*
3190 * Update the multicast addresses to the MTA registers
3191 */
3192 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3193 ixgbe_mc_table_itr, TRUE);
3194 }
3195
3196 /*
3197 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3198 *
3199 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3200 * Different chipsets may have different allowed configuration of vmdq and rss.
3201 */
3202 static void
3203 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3204 {
3205 struct ixgbe_hw *hw = &ixgbe->hw;
3206 uint32_t ring_per_group;
3207
3208 switch (hw->mac.type) {
3209 case ixgbe_mac_82598EB:
3210 /*
3211 * 82598 supports the following combination:
3212 * vmdq no. x rss no.
3213 * [5..16] x 1
3214 * [1..4] x [1..16]
3215 * However 8 rss queue per pool (vmdq) is sufficient for
3216 * most cases.
3217 */
3218 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3219 if (ixgbe->num_rx_groups > 4) {
3220 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3221 } else {
3222 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3223 min(8, ring_per_group);
3224 }
3225
3226 break;
3227
3228 case ixgbe_mac_82599EB:
3229 case ixgbe_mac_X540:
3230 case ixgbe_mac_X550:
3231 case ixgbe_mac_X550EM_x:
3232 /*
3233 * 82599 supports the following combination:
3234 * vmdq no. x rss no.
3235 * [33..64] x [1..2]
3236 * [2..32] x [1..4]
3237 * 1 x [1..16]
3238 * However 8 rss queue per pool (vmdq) is sufficient for
3239 * most cases.
3240 *
3241 * For now, treat X540 and X550 like the 82599.
3242 */
3243 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3244 if (ixgbe->num_rx_groups == 1) {
3245 ixgbe->num_rx_rings = min(8, ring_per_group);
3246 } else if (ixgbe->num_rx_groups <= 32) {
3247 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3248 min(4, ring_per_group);
3249 } else if (ixgbe->num_rx_groups <= 64) {
3250 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3251 min(2, ring_per_group);
3252 }
3253 break;
3254
3255 default:
3256 break;
3257 }
3258
3259 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3260
3261 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3262 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3263 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3264 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3265 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3266 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3267 } else {
3268 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3269 }
3270
3271 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3272 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3273 }
3274
3275 /*
3276 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3277 *
3278 * This routine gets user-configured values out of the configuration
3279 * file ixgbe.conf.
3280 *
3281 * For each configurable value, there is a minimum, a maximum, and a
3282 * default.
3283 * If user does not configure a value, use the default.
3284 * If user configures below the minimum, use the minumum.
3285 * If user configures above the maximum, use the maxumum.
3286 */
3287 static void
3288 ixgbe_get_conf(ixgbe_t *ixgbe)
3289 {
3290 struct ixgbe_hw *hw = &ixgbe->hw;
3291 uint32_t flow_control;
3292
3293 /*
3294 * ixgbe driver supports the following user configurations:
3295 *
3296 * Jumbo frame configuration:
3297 * default_mtu
3298 *
3299 * Ethernet flow control configuration:
3300 * flow_control
3301 *
3302 * Multiple rings configurations:
3303 * tx_queue_number
3304 * tx_ring_size
3305 * rx_queue_number
3306 * rx_ring_size
3307 *
3308 * Call ixgbe_get_prop() to get the value for a specific
3309 * configuration parameter.
3310 */
3311
3312 /*
3313 * Jumbo frame configuration - max_frame_size controls host buffer
3314 * allocation, so includes MTU, ethernet header, vlan tag and
3315 * frame check sequence.
3316 */
3317 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3318 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3319
3320 ixgbe->max_frame_size = ixgbe->default_mtu +
3321 sizeof (struct ether_vlan_header) + ETHERFCSL;
3322
3323 /*
3324 * Ethernet flow control configuration
3325 */
3326 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3327 ixgbe_fc_none, 3, ixgbe_fc_none);
3328 if (flow_control == 3)
3329 flow_control = ixgbe_fc_default;
3330
3331 /*
3332 * fc.requested mode is what the user requests. After autoneg,
3333 * fc.current_mode will be the flow_control mode that was negotiated.
3334 */
3335 hw->fc.requested_mode = flow_control;
3336
3337 /*
3338 * Multiple rings configurations
3339 */
3340 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3341 ixgbe->capab->min_tx_que_num,
3342 ixgbe->capab->max_tx_que_num,
3343 ixgbe->capab->def_tx_que_num);
3344 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3345 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3346
3347 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3348 ixgbe->capab->min_rx_que_num,
3349 ixgbe->capab->max_rx_que_num,
3350 ixgbe->capab->def_rx_que_num);
3351 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3352 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3353
3354 /*
3355 * Multiple groups configuration
3356 */
3357 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3358 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3359 ixgbe->capab->def_rx_grp_num);
3360
3361 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3362 0, 1, DEFAULT_MR_ENABLE);
3363
3364 if (ixgbe->mr_enable == B_FALSE) {
3365 ixgbe->num_tx_rings = 1;
3366 ixgbe->num_rx_rings = 1;
3367 ixgbe->num_rx_groups = 1;
3368 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3369 } else {
3370 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3371 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3372 /*
3373 * The combination of num_rx_rings and num_rx_groups
3374 * may be not supported by h/w. We need to adjust
3375 * them to appropriate values.
3376 */
3377 ixgbe_setup_vmdq_rss_conf(ixgbe);
3378 }
3379
3380 /*
3381 * Tunable used to force an interrupt type. The only use is
3382 * for testing of the lesser interrupt types.
3383 * 0 = don't force interrupt type
3384 * 1 = force interrupt type MSI-X
3385 * 2 = force interrupt type MSI
3386 * 3 = force interrupt type Legacy
3387 */
3388 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3389 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3390
3391 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3392 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3393 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3394 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3395 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3396 0, 1, DEFAULT_LSO_ENABLE);
3397 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3398 0, 1, DEFAULT_LRO_ENABLE);
3399 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3400 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3401 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3402 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3403
3404 /* Head Write Back not recommended for 82599, X540 and X550 */
3405 if (hw->mac.type == ixgbe_mac_82599EB ||
3406 hw->mac.type == ixgbe_mac_X540 ||
3407 hw->mac.type == ixgbe_mac_X550 ||
3408 hw->mac.type == ixgbe_mac_X550EM_x) {
3409 ixgbe->tx_head_wb_enable = B_FALSE;
3410 }
3411
3412 /*
3413 * ixgbe LSO needs the tx h/w checksum support.
3414 * LSO will be disabled if tx h/w checksum is not
3415 * enabled.
3416 */
3417 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3418 ixgbe->lso_enable = B_FALSE;
3419 }
3420
3421 /*
3422 * ixgbe LRO needs the rx h/w checksum support.
3423 * LRO will be disabled if rx h/w checksum is not
3424 * enabled.
3425 */
3426 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3427 ixgbe->lro_enable = B_FALSE;
3428 }
3429
3430 /*
3431 * ixgbe LRO only supported by 82599, X540 and X550
3432 */
3433 if (hw->mac.type == ixgbe_mac_82598EB) {
3434 ixgbe->lro_enable = B_FALSE;
3435 }
3436 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3437 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3438 DEFAULT_TX_COPY_THRESHOLD);
3439 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3440 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3441 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3442 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3443 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3444 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3445 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3446 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3447 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3448
3449 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3450 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3451 DEFAULT_RX_COPY_THRESHOLD);
3452 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3453 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3454 DEFAULT_RX_LIMIT_PER_INTR);
3455
3456 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3457 ixgbe->capab->min_intr_throttle,
3458 ixgbe->capab->max_intr_throttle,
3459 ixgbe->capab->def_intr_throttle);
3460 /*
3461 * 82599, X540 and X550 require the interrupt throttling rate is
3462 * a multiple of 8. This is enforced by the register definiton.
3463 */
3464 if (hw->mac.type == ixgbe_mac_82599EB ||
3465 hw->mac.type == ixgbe_mac_X540 ||
3466 hw->mac.type == ixgbe_mac_X550 ||
3467 hw->mac.type == ixgbe_mac_X550EM_x)
3468 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3469
3470 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3471 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3472 }
3473
3474 static void
3475 ixgbe_init_params(ixgbe_t *ixgbe)
3476 {
3477 struct ixgbe_hw *hw = &ixgbe->hw;
3478 ixgbe_link_speed speeds_supported = 0;
3479 boolean_t negotiate;
3480
3481 /*
3482 * Get a list of speeds the adapter supports. If the hw struct hasn't
3483 * been populated with this information yet, retrieve it from the
3484 * adapter and save it to our own variable.
3485 *
3486 * On certain adapters, such as ones which use SFPs, the contents of
3487 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3488 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3489 * in order to ascertain the speeds which we are capable of supporting,
3490 * and in the case of SFP-equipped adapters, which speed we are
3491 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3492 * we'll go with a default list of speeds as a last resort.
3493 */
3494 speeds_supported = hw->phy.speeds_supported;
3495
3496 if (speeds_supported == 0) {
3497 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3498 &negotiate) != IXGBE_SUCCESS) {
3499 if (hw->mac.type == ixgbe_mac_82598EB) {
3500 speeds_supported =
3501 IXGBE_LINK_SPEED_82598_AUTONEG;
3502 } else {
3503 speeds_supported =
3504 IXGBE_LINK_SPEED_82599_AUTONEG;
3505 }
3506 }
3507 }
3508 ixgbe->speeds_supported = speeds_supported;
3509
3510 /*
3511 * By default, all supported speeds are enabled and advertised.
3512 */
3513 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3514 ixgbe->param_en_10000fdx_cap = 1;
3515 ixgbe->param_adv_10000fdx_cap = 1;
3516 } else {
3517 ixgbe->param_en_10000fdx_cap = 0;
3518 ixgbe->param_adv_10000fdx_cap = 0;
3519 }
3520
3521 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3522 ixgbe->param_en_5000fdx_cap = 1;
3523 ixgbe->param_adv_5000fdx_cap = 1;
3524 } else {
3525 ixgbe->param_en_5000fdx_cap = 0;
3526 ixgbe->param_adv_5000fdx_cap = 0;
3527 }
3528
3529 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3530 ixgbe->param_en_2500fdx_cap = 1;
3531 ixgbe->param_adv_2500fdx_cap = 1;
3532 } else {
3533 ixgbe->param_en_2500fdx_cap = 0;
3534 ixgbe->param_adv_2500fdx_cap = 0;
3535 }
3536
3537 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3538 ixgbe->param_en_1000fdx_cap = 1;
3539 ixgbe->param_adv_1000fdx_cap = 1;
3540 } else {
3541 ixgbe->param_en_1000fdx_cap = 0;
3542 ixgbe->param_adv_1000fdx_cap = 0;
3543 }
3544
3545 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3546 ixgbe->param_en_100fdx_cap = 1;
3547 ixgbe->param_adv_100fdx_cap = 1;
3548 } else {
3549 ixgbe->param_en_100fdx_cap = 0;
3550 ixgbe->param_adv_100fdx_cap = 0;
3551 }
3552
3553 ixgbe->param_pause_cap = 1;
3554 ixgbe->param_asym_pause_cap = 1;
3555 ixgbe->param_rem_fault = 0;
3556
3557 ixgbe->param_adv_autoneg_cap = 1;
3558 ixgbe->param_adv_pause_cap = 1;
3559 ixgbe->param_adv_asym_pause_cap = 1;
3560 ixgbe->param_adv_rem_fault = 0;
3561
3562 ixgbe->param_lp_10000fdx_cap = 0;
3563 ixgbe->param_lp_5000fdx_cap = 0;
3564 ixgbe->param_lp_2500fdx_cap = 0;
3565 ixgbe->param_lp_1000fdx_cap = 0;
3566 ixgbe->param_lp_100fdx_cap = 0;
3567 ixgbe->param_lp_autoneg_cap = 0;
3568 ixgbe->param_lp_pause_cap = 0;
3569 ixgbe->param_lp_asym_pause_cap = 0;
3570 ixgbe->param_lp_rem_fault = 0;
3571 }
3572
3573 /*
3574 * ixgbe_get_prop - Get a property value out of the configuration file
3575 * ixgbe.conf.
3576 *
3577 * Caller provides the name of the property, a default value, a minimum
3578 * value, and a maximum value.
3579 *
3580 * Return configured value of the property, with default, minimum and
3581 * maximum properly applied.
3582 */
3583 static int
3584 ixgbe_get_prop(ixgbe_t *ixgbe,
3585 char *propname, /* name of the property */
3586 int minval, /* minimum acceptable value */
3587 int maxval, /* maximim acceptable value */
3588 int defval) /* default value */
3589 {
3590 int value;
3591
3592 /*
3593 * Call ddi_prop_get_int() to read the conf settings
3594 */
3595 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3596 DDI_PROP_DONTPASS, propname, defval);
3597 if (value > maxval)
3598 value = maxval;
3599
3600 if (value < minval)
3601 value = minval;
3602
3603 return (value);
3604 }
3605
3606 /*
3607 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3608 */
3609 int
3610 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3611 {
3612 struct ixgbe_hw *hw = &ixgbe->hw;
3613 ixgbe_link_speed advertised = 0;
3614
3615 /*
3616 * Assemble a list of enabled speeds to auto-negotiate with.
3617 */
3618 if (ixgbe->param_en_10000fdx_cap == 1)
3619 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3620
3621 if (ixgbe->param_en_5000fdx_cap == 1)
3622 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3623
3624 if (ixgbe->param_en_2500fdx_cap == 1)
3625 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3626
3627 if (ixgbe->param_en_1000fdx_cap == 1)
3628 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3629
3630 if (ixgbe->param_en_100fdx_cap == 1)
3631 advertised |= IXGBE_LINK_SPEED_100_FULL;
3632
3633 /*
3634 * As a last resort, autoneg with a default list of speeds.
3635 */
3636 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3637 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3638 "to autonegotiate with full capabilities.");
3639
3640 if (hw->mac.type == ixgbe_mac_82598EB)
3641 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3642 else
3643 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3644 }
3645
3646 if (setup_hw) {
3647 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3648 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3649 ixgbe_notice(ixgbe, "Setup link failed on this "
3650 "device.");
3651 return (IXGBE_FAILURE);
3652 }
3653 }
3654
3655 return (IXGBE_SUCCESS);
3656 }
3657
3658 /*
3659 * ixgbe_driver_link_check - Link status processing.
3660 *
3661 * This function can be called in both kernel context and interrupt context
3662 */
3663 static void
3664 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3665 {
3666 struct ixgbe_hw *hw = &ixgbe->hw;
3667 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3668 boolean_t link_up = B_FALSE;
3669 boolean_t link_changed = B_FALSE;
3670
3671 ASSERT(mutex_owned(&ixgbe->gen_lock));
3672
3673 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3674 if (link_up) {
3675 ixgbe->link_check_complete = B_TRUE;
3676
3677 /* Link is up, enable flow control settings */
3678 (void) ixgbe_fc_enable(hw);
3679
3680 /*
3681 * The Link is up, check whether it was marked as down earlier
3682 */
3683 if (ixgbe->link_state != LINK_STATE_UP) {
3684 switch (speed) {
3685 case IXGBE_LINK_SPEED_10GB_FULL:
3686 ixgbe->link_speed = SPEED_10GB;
3687 break;
3688 case IXGBE_LINK_SPEED_5GB_FULL:
3689 ixgbe->link_speed = SPEED_5GB;
3690 break;
3691 case IXGBE_LINK_SPEED_2_5GB_FULL:
3692 ixgbe->link_speed = SPEED_2_5GB;
3693 break;
3694 case IXGBE_LINK_SPEED_1GB_FULL:
3695 ixgbe->link_speed = SPEED_1GB;
3696 break;
3697 case IXGBE_LINK_SPEED_100_FULL:
3698 ixgbe->link_speed = SPEED_100;
3699 }
3700 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3701 ixgbe->link_state = LINK_STATE_UP;
3702 link_changed = B_TRUE;
3703 }
3704 } else {
3705 if (ixgbe->link_check_complete == B_TRUE ||
3706 (ixgbe->link_check_complete == B_FALSE &&
3707 gethrtime() >= ixgbe->link_check_hrtime)) {
3708 /*
3709 * The link is really down
3710 */
3711 ixgbe->link_check_complete = B_TRUE;
3712
3713 if (ixgbe->link_state != LINK_STATE_DOWN) {
3714 ixgbe->link_speed = 0;
3715 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3716 ixgbe->link_state = LINK_STATE_DOWN;
3717 link_changed = B_TRUE;
3718 }
3719 }
3720 }
3721
3722 /*
3723 * If we are in an interrupt context, need to re-enable the
3724 * interrupt, which was automasked
3725 */
3726 if (servicing_interrupt() != 0) {
3727 ixgbe->eims |= IXGBE_EICR_LSC;
3728 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3729 }
3730
3731 if (link_changed) {
3732 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3733 }
3734 }
3735
3736 /*
3737 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3738 */
3739 static void
3740 ixgbe_sfp_check(void *arg)
3741 {
3742 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3743 uint32_t eicr = ixgbe->eicr;
3744 struct ixgbe_hw *hw = &ixgbe->hw;
3745
3746 mutex_enter(&ixgbe->gen_lock);
3747 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3748 /* clear the interrupt */
3749 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3750
3751 /* if link up, do multispeed fiber setup */
3752 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3753 B_TRUE);
3754 ixgbe_driver_link_check(ixgbe);
3755 ixgbe_get_hw_state(ixgbe);
3756 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3757 /* clear the interrupt */
3758 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3759
3760 /* if link up, do sfp module setup */
3761 (void) hw->mac.ops.setup_sfp(hw);
3762
3763 /* do multispeed fiber setup */
3764 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3765 B_TRUE);
3766 ixgbe_driver_link_check(ixgbe);
3767 ixgbe_get_hw_state(ixgbe);
3768 }
3769 mutex_exit(&ixgbe->gen_lock);
3770
3771 /*
3772 * We need to fully re-check the link later.
3773 */
3774 ixgbe->link_check_complete = B_FALSE;
3775 ixgbe->link_check_hrtime = gethrtime() +
3776 (IXGBE_LINK_UP_TIME * 100000000ULL);
3777 }
3778
3779 /*
3780 * ixgbe_overtemp_check - overtemp module processing done in taskq
3781 *
3782 * This routine will only be called on adapters with temperature sensor.
3783 * The indication of over-temperature can be either SDP0 interrupt or the link
3784 * status change interrupt.
3785 */
3786 static void
3787 ixgbe_overtemp_check(void *arg)
3788 {
3789 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3790 struct ixgbe_hw *hw = &ixgbe->hw;
3791 uint32_t eicr = ixgbe->eicr;
3792 ixgbe_link_speed speed;
3793 boolean_t link_up;
3794
3795 mutex_enter(&ixgbe->gen_lock);
3796
3797 /* make sure we know current state of link */
3798 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3799
3800 /* check over-temp condition */
3801 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3802 (eicr & IXGBE_EICR_LSC)) {
3803 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3804 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3805
3806 /*
3807 * Disable the adapter interrupts
3808 */
3809 ixgbe_disable_adapter_interrupts(ixgbe);
3810
3811 /*
3812 * Disable Rx/Tx units
3813 */
3814 (void) ixgbe_stop_adapter(hw);
3815
3816 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3817 ixgbe_error(ixgbe,
3818 "Problem: Network adapter has been stopped "
3819 "because it has overheated");
3820 ixgbe_error(ixgbe,
3821 "Action: Restart the computer. "
3822 "If the problem persists, power off the system "
3823 "and replace the adapter");
3824 }
3825 }
3826
3827 /* write to clear the interrupt */
3828 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3829
3830 mutex_exit(&ixgbe->gen_lock);
3831 }
3832
3833 /*
3834 * ixgbe_phy_check - taskq to process interrupts from an external PHY
3835 *
3836 * This routine will only be called on adapters with external PHYs
3837 * (such as X550) that may be trying to raise our attention to some event.
3838 * Currently, this is limited to claiming PHY overtemperature and link status
3839 * change (LSC) events, however this may expand to include other things in
3840 * future adapters.
3841 */
3842 static void
3843 ixgbe_phy_check(void *arg)
3844 {
3845 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3846 struct ixgbe_hw *hw = &ixgbe->hw;
3847 int rv;
3848
3849 mutex_enter(&ixgbe->gen_lock);
3850
3851 /*
3852 * X550 baseT PHY overtemp and LSC events are handled here.
3853 *
3854 * If an overtemp event occurs, it will be reflected in the
3855 * return value of phy.ops.handle_lasi() and the common code will
3856 * automatically power off the baseT PHY. This is our cue to trigger
3857 * an FMA event.
3858 *
3859 * If a link status change event occurs, phy.ops.handle_lasi() will
3860 * automatically initiate a link setup between the integrated KR PHY
3861 * and the external X557 PHY to ensure that the link speed between
3862 * them matches the link speed of the baseT link.
3863 */
3864 rv = ixgbe_handle_lasi(hw);
3865
3866 if (rv == IXGBE_ERR_OVERTEMP) {
3867 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3868
3869 /*
3870 * Disable the adapter interrupts
3871 */
3872 ixgbe_disable_adapter_interrupts(ixgbe);
3873
3874 /*
3875 * Disable Rx/Tx units
3876 */
3877 (void) ixgbe_stop_adapter(hw);
3878
3879 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3880 ixgbe_error(ixgbe,
3881 "Problem: Network adapter has been stopped due to a "
3882 "overtemperature event being detected.");
3883 ixgbe_error(ixgbe,
3884 "Action: Shut down or restart the computer. If the issue "
3885 "persists, please take action in accordance with the "
3886 "recommendations from your system vendor.");
3887 }
3888
3889 mutex_exit(&ixgbe->gen_lock);
3890 }
3891
3892 /*
3893 * ixgbe_link_timer - timer for link status detection
3894 */
3895 static void
3896 ixgbe_link_timer(void *arg)
3897 {
3898 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3899
3900 mutex_enter(&ixgbe->gen_lock);
3901 ixgbe_driver_link_check(ixgbe);
3902 mutex_exit(&ixgbe->gen_lock);
3903 }
3904
3905 /*
3906 * ixgbe_local_timer - Driver watchdog function.
3907 *
3908 * This function will handle the transmit stall check and other routines.
3909 */
3910 static void
3911 ixgbe_local_timer(void *arg)
3912 {
3913 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3914
3915 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3916 goto out;
3917
3918 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3919 ixgbe->reset_count++;
3920 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3921 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3922 goto out;
3923 }
3924
3925 if (ixgbe_stall_check(ixgbe)) {
3926 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3927 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3928
3929 ixgbe->reset_count++;
3930 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3931 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3932 }
3933
3934 out:
3935 ixgbe_restart_watchdog_timer(ixgbe);
3936 }
3937
3938 /*
3939 * ixgbe_stall_check - Check for transmit stall.
3940 *
3941 * This function checks if the adapter is stalled (in transmit).
3942 *
3943 * It is called each time the watchdog timeout is invoked.
3944 * If the transmit descriptor reclaim continuously fails,
3945 * the watchdog value will increment by 1. If the watchdog
3946 * value exceeds the threshold, the ixgbe is assumed to
3947 * have stalled and need to be reset.
3948 */
3949 static boolean_t
3950 ixgbe_stall_check(ixgbe_t *ixgbe)
3951 {
3952 ixgbe_tx_ring_t *tx_ring;
3953 boolean_t result;
3954 int i;
3955
3956 if (ixgbe->link_state != LINK_STATE_UP)
3957 return (B_FALSE);
3958
3959 /*
3960 * If any tx ring is stalled, we'll reset the chipset
3961 */
3962 result = B_FALSE;
3963 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3964 tx_ring = &ixgbe->tx_rings[i];
3965 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3966 tx_ring->tx_recycle(tx_ring);
3967 }
3968
3969 if (tx_ring->recycle_fail > 0)
3970 tx_ring->stall_watchdog++;
3971 else
3972 tx_ring->stall_watchdog = 0;
3973
3974 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3975 result = B_TRUE;
3976 break;
3977 }
3978 }
3979
3980 if (result) {
3981 tx_ring->stall_watchdog = 0;
3982 tx_ring->recycle_fail = 0;
3983 }
3984
3985 return (result);
3986 }
3987
3988
3989 /*
3990 * is_valid_mac_addr - Check if the mac address is valid.
3991 */
3992 static boolean_t
3993 is_valid_mac_addr(uint8_t *mac_addr)
3994 {
3995 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3996 const uint8_t addr_test2[6] =
3997 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3998
3999 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4000 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4001 return (B_FALSE);
4002
4003 return (B_TRUE);
4004 }
4005
4006 static boolean_t
4007 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4008 {
4009 #ifdef __sparc
4010 struct ixgbe_hw *hw = &ixgbe->hw;
4011 uchar_t *bytes;
4012 struct ether_addr sysaddr;
4013 uint_t nelts;
4014 int err;
4015 boolean_t found = B_FALSE;
4016
4017 /*
4018 * The "vendor's factory-set address" may already have
4019 * been extracted from the chip, but if the property
4020 * "local-mac-address" is set we use that instead.
4021 *
4022 * We check whether it looks like an array of 6
4023 * bytes (which it should, if OBP set it). If we can't
4024 * make sense of it this way, we'll ignore it.
4025 */
4026 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4027 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4028 if (err == DDI_PROP_SUCCESS) {
4029 if (nelts == ETHERADDRL) {
4030 while (nelts--)
4031 hw->mac.addr[nelts] = bytes[nelts];
4032 found = B_TRUE;
4033 }
4034 ddi_prop_free(bytes);
4035 }
4036
4037 /*
4038 * Look up the OBP property "local-mac-address?". If the user has set
4039 * 'local-mac-address? = false', use "the system address" instead.
4040 */
4041 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4042 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4043 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4044 if (localetheraddr(NULL, &sysaddr) != 0) {
4045 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4046 found = B_TRUE;
4047 }
4048 }
4049 ddi_prop_free(bytes);
4050 }
4051
4052 /*
4053 * Finally(!), if there's a valid "mac-address" property (created
4054 * if we netbooted from this interface), we must use this instead
4055 * of any of the above to ensure that the NFS/install server doesn't
4056 * get confused by the address changing as illumos takes over!
4057 */
4058 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4059 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4060 if (err == DDI_PROP_SUCCESS) {
4061 if (nelts == ETHERADDRL) {
4062 while (nelts--)
4063 hw->mac.addr[nelts] = bytes[nelts];
4064 found = B_TRUE;
4065 }
4066 ddi_prop_free(bytes);
4067 }
4068
4069 if (found) {
4070 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4071 return (B_TRUE);
4072 }
4073 #else
4074 _NOTE(ARGUNUSED(ixgbe));
4075 #endif
4076
4077 return (B_TRUE);
4078 }
4079
4080 #pragma inline(ixgbe_arm_watchdog_timer)
4081 static void
4082 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4083 {
4084 /*
4085 * Fire a watchdog timer
4086 */
4087 ixgbe->watchdog_tid =
4088 timeout(ixgbe_local_timer,
4089 (void *)ixgbe, 1 * drv_usectohz(1000000));
4090
4091 }
4092
4093 /*
4094 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4095 */
4096 void
4097 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4098 {
4099 mutex_enter(&ixgbe->watchdog_lock);
4100
4101 if (!ixgbe->watchdog_enable) {
4102 ixgbe->watchdog_enable = B_TRUE;
4103 ixgbe->watchdog_start = B_TRUE;
4104 ixgbe_arm_watchdog_timer(ixgbe);
4105 }
4106
4107 mutex_exit(&ixgbe->watchdog_lock);
4108 }
4109
4110 /*
4111 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4112 */
4113 void
4114 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4115 {
4116 timeout_id_t tid;
4117
4118 mutex_enter(&ixgbe->watchdog_lock);
4119
4120 ixgbe->watchdog_enable = B_FALSE;
4121 ixgbe->watchdog_start = B_FALSE;
4122 tid = ixgbe->watchdog_tid;
4123 ixgbe->watchdog_tid = 0;
4124
4125 mutex_exit(&ixgbe->watchdog_lock);
4126
4127 if (tid != 0)
4128 (void) untimeout(tid);
4129 }
4130
4131 /*
4132 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4133 */
4134 void
4135 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4136 {
4137 mutex_enter(&ixgbe->watchdog_lock);
4138
4139 if (ixgbe->watchdog_enable) {
4140 if (!ixgbe->watchdog_start) {
4141 ixgbe->watchdog_start = B_TRUE;
4142 ixgbe_arm_watchdog_timer(ixgbe);
4143 }
4144 }
4145
4146 mutex_exit(&ixgbe->watchdog_lock);
4147 }
4148
4149 /*
4150 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4151 */
4152 static void
4153 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4154 {
4155 mutex_enter(&ixgbe->watchdog_lock);
4156
4157 if (ixgbe->watchdog_start)
4158 ixgbe_arm_watchdog_timer(ixgbe);
4159
4160 mutex_exit(&ixgbe->watchdog_lock);
4161 }
4162
4163 /*
4164 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4165 */
4166 void
4167 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4168 {
4169 timeout_id_t tid;
4170
4171 mutex_enter(&ixgbe->watchdog_lock);
4172
4173 ixgbe->watchdog_start = B_FALSE;
4174 tid = ixgbe->watchdog_tid;
4175 ixgbe->watchdog_tid = 0;
4176
4177 mutex_exit(&ixgbe->watchdog_lock);
4178
4179 if (tid != 0)
4180 (void) untimeout(tid);
4181 }
4182
4183 /*
4184 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4185 */
4186 static void
4187 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4188 {
4189 struct ixgbe_hw *hw = &ixgbe->hw;
4190
4191 /*
4192 * mask all interrupts off
4193 */
4194 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4195
4196 /*
4197 * for MSI-X, also disable autoclear
4198 */
4199 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4200 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4201 }
4202
4203 IXGBE_WRITE_FLUSH(hw);
4204 }
4205
4206 /*
4207 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4208 */
4209 static void
4210 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4211 {
4212 struct ixgbe_hw *hw = &ixgbe->hw;
4213 uint32_t eiac, eiam;
4214 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4215
4216 /* interrupt types to enable */
4217 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4218 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4219 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4220
4221 /* enable automask on "other" causes that this adapter can generate */
4222 eiam = ixgbe->capab->other_intr;
4223
4224 /*
4225 * msi-x mode
4226 */
4227 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4228 /* enable autoclear but not on bits 29:20 */
4229 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4230
4231 /* general purpose interrupt enable */
4232 gpie |= (IXGBE_GPIE_MSIX_MODE
4233 | IXGBE_GPIE_PBA_SUPPORT
4234 | IXGBE_GPIE_OCD
4235 | IXGBE_GPIE_EIAME);
4236 /*
4237 * non-msi-x mode
4238 */
4239 } else {
4240
4241 /* disable autoclear, leave gpie at default */
4242 eiac = 0;
4243
4244 /*
4245 * General purpose interrupt enable.
4246 * For 82599, X540 and X550, extended interrupt
4247 * automask enable only in MSI or MSI-X mode
4248 */
4249 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4250 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4251 gpie |= IXGBE_GPIE_EIAME;
4252 }
4253 }
4254
4255 /* Enable specific "other" interrupt types */
4256 switch (hw->mac.type) {
4257 case ixgbe_mac_82598EB:
4258 gpie |= ixgbe->capab->other_gpie;
4259 break;
4260
4261 case ixgbe_mac_82599EB:
4262 case ixgbe_mac_X540:
4263 case ixgbe_mac_X550:
4264 case ixgbe_mac_X550EM_x:
4265 gpie |= ixgbe->capab->other_gpie;
4266
4267 /* Enable RSC Delay 8us when LRO enabled */
4268 if (ixgbe->lro_enable) {
4269 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4270 }
4271 break;
4272
4273 default:
4274 break;
4275 }
4276
4277 /* write to interrupt control registers */
4278 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4279 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4280 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4281 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4282 IXGBE_WRITE_FLUSH(hw);
4283 }
4284
4285 /*
4286 * ixgbe_loopback_ioctl - Loopback support.
4287 */
4288 enum ioc_reply
4289 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4290 {
4291 lb_info_sz_t *lbsp;
4292 lb_property_t *lbpp;
4293 uint32_t *lbmp;
4294 uint32_t size;
4295 uint32_t value;
4296
4297 if (mp->b_cont == NULL)
4298 return (IOC_INVAL);
4299
4300 switch (iocp->ioc_cmd) {
4301 default:
4302 return (IOC_INVAL);
4303
4304 case LB_GET_INFO_SIZE:
4305 size = sizeof (lb_info_sz_t);
4306 if (iocp->ioc_count != size)
4307 return (IOC_INVAL);
4308
4309 value = sizeof (lb_normal);
4310 value += sizeof (lb_mac);
4311 value += sizeof (lb_external);
4312
4313 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4314 *lbsp = value;
4315 break;
4316
4317 case LB_GET_INFO:
4318 value = sizeof (lb_normal);
4319 value += sizeof (lb_mac);
4320 value += sizeof (lb_external);
4321
4322 size = value;
4323 if (iocp->ioc_count != size)
4324 return (IOC_INVAL);
4325
4326 value = 0;
4327 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4328
4329 lbpp[value++] = lb_normal;
4330 lbpp[value++] = lb_mac;
4331 lbpp[value++] = lb_external;
4332 break;
4333
4334 case LB_GET_MODE:
4335 size = sizeof (uint32_t);
4336 if (iocp->ioc_count != size)
4337 return (IOC_INVAL);
4338
4339 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4340 *lbmp = ixgbe->loopback_mode;
4341 break;
4342
4343 case LB_SET_MODE:
4344 size = 0;
4345 if (iocp->ioc_count != sizeof (uint32_t))
4346 return (IOC_INVAL);
4347
4348 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4349 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4350 return (IOC_INVAL);
4351 break;
4352 }
4353
4354 iocp->ioc_count = size;
4355 iocp->ioc_error = 0;
4356
4357 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4358 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4359 return (IOC_INVAL);
4360 }
4361
4362 return (IOC_REPLY);
4363 }
4364
4365 /*
4366 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4367 */
4368 static boolean_t
4369 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4370 {
4371 if (mode == ixgbe->loopback_mode)
4372 return (B_TRUE);
4373
4374 ixgbe->loopback_mode = mode;
4375
4376 if (mode == IXGBE_LB_NONE) {
4377 /*
4378 * Reset the chip
4379 */
4380 (void) ixgbe_reset(ixgbe);
4381 return (B_TRUE);
4382 }
4383
4384 mutex_enter(&ixgbe->gen_lock);
4385
4386 switch (mode) {
4387 default:
4388 mutex_exit(&ixgbe->gen_lock);
4389 return (B_FALSE);
4390
4391 case IXGBE_LB_EXTERNAL:
4392 break;
4393
4394 case IXGBE_LB_INTERNAL_MAC:
4395 ixgbe_set_internal_mac_loopback(ixgbe);
4396 break;
4397 }
4398
4399 mutex_exit(&ixgbe->gen_lock);
4400
4401 return (B_TRUE);
4402 }
4403
4404 /*
4405 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4406 */
4407 static void
4408 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4409 {
4410 struct ixgbe_hw *hw;
4411 uint32_t reg;
4412 uint8_t atlas;
4413
4414 hw = &ixgbe->hw;
4415
4416 /*
4417 * Setup MAC loopback
4418 */
4419 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4420 reg |= IXGBE_HLREG0_LPBK;
4421 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4422
4423 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4424 reg &= ~IXGBE_AUTOC_LMS_MASK;
4425 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4426
4427 /*
4428 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4429 */
4430 switch (hw->mac.type) {
4431 case ixgbe_mac_82598EB:
4432 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4433 &atlas);
4434 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4435 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4436 atlas);
4437
4438 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4439 &atlas);
4440 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4441 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4442 atlas);
4443
4444 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4445 &atlas);
4446 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4447 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4448 atlas);
4449
4450 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4451 &atlas);
4452 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4453 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4454 atlas);
4455 break;
4456
4457 case ixgbe_mac_82599EB:
4458 case ixgbe_mac_X540:
4459 case ixgbe_mac_X550:
4460 case ixgbe_mac_X550EM_x:
4461 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4462 reg |= (IXGBE_AUTOC_FLU |
4463 IXGBE_AUTOC_10G_KX4);
4464 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4465
4466 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4467 B_FALSE);
4468 break;
4469
4470 default:
4471 break;
4472 }
4473 }
4474
4475 #pragma inline(ixgbe_intr_rx_work)
4476 /*
4477 * ixgbe_intr_rx_work - RX processing of ISR.
4478 */
4479 static void
4480 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4481 {
4482 mblk_t *mp;
4483
4484 mutex_enter(&rx_ring->rx_lock);
4485
4486 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4487 mutex_exit(&rx_ring->rx_lock);
4488
4489 if (mp != NULL)
4490 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4491 rx_ring->ring_gen_num);
4492 }
4493
4494 #pragma inline(ixgbe_intr_tx_work)
4495 /*
4496 * ixgbe_intr_tx_work - TX processing of ISR.
4497 */
4498 static void
4499 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4500 {
4501 ixgbe_t *ixgbe = tx_ring->ixgbe;
4502
4503 /*
4504 * Recycle the tx descriptors
4505 */
4506 tx_ring->tx_recycle(tx_ring);
4507
4508 /*
4509 * Schedule the re-transmit
4510 */
4511 if (tx_ring->reschedule &&
4512 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4513 tx_ring->reschedule = B_FALSE;
4514 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4515 tx_ring->ring_handle);
4516 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4517 }
4518 }
4519
4520 #pragma inline(ixgbe_intr_other_work)
4521 /*
4522 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4523 */
4524 static void
4525 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4526 {
4527 struct ixgbe_hw *hw = &ixgbe->hw;
4528
4529 ASSERT(mutex_owned(&ixgbe->gen_lock));
4530
4531 /*
4532 * handle link status change
4533 */
4534 if (eicr & IXGBE_EICR_LSC) {
4535 ixgbe_driver_link_check(ixgbe);
4536 ixgbe_get_hw_state(ixgbe);
4537 }
4538
4539 /*
4540 * check for fan failure on adapters with fans
4541 */
4542 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4543 (eicr & IXGBE_EICR_GPI_SDP1)) {
4544 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4545
4546 /*
4547 * Disable the adapter interrupts
4548 */
4549 ixgbe_disable_adapter_interrupts(ixgbe);
4550
4551 /*
4552 * Disable Rx/Tx units
4553 */
4554 (void) ixgbe_stop_adapter(&ixgbe->hw);
4555
4556 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4557 ixgbe_error(ixgbe,
4558 "Problem: Network adapter has been stopped "
4559 "because the fan has stopped.\n");
4560 ixgbe_error(ixgbe,
4561 "Action: Replace the adapter.\n");
4562
4563 /* re-enable the interrupt, which was automasked */
4564 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4565 }
4566
4567 /*
4568 * Do SFP check for adapters with hot-plug capability
4569 */
4570 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4571 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4572 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4573 ixgbe->eicr = eicr;
4574 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4575 ixgbe_sfp_check, (void *)ixgbe,
4576 DDI_NOSLEEP)) != DDI_SUCCESS) {
4577 ixgbe_log(ixgbe, "No memory available to dispatch "
4578 "taskq for SFP check");
4579 }
4580 }
4581
4582 /*
4583 * Do over-temperature check for adapters with temp sensor
4584 */
4585 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4586 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4587 (eicr & IXGBE_EICR_LSC))) {
4588 ixgbe->eicr = eicr;
4589 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4590 ixgbe_overtemp_check, (void *)ixgbe,
4591 DDI_NOSLEEP)) != DDI_SUCCESS) {
4592 ixgbe_log(ixgbe, "No memory available to dispatch "
4593 "taskq for overtemp check");
4594 }
4595 }
4596
4597 /*
4598 * Process an external PHY interrupt
4599 */
4600 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4601 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4602 ixgbe->eicr = eicr;
4603 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4604 ixgbe_phy_check, (void *)ixgbe,
4605 DDI_NOSLEEP)) != DDI_SUCCESS) {
4606 ixgbe_log(ixgbe, "No memory available to dispatch "
4607 "taskq for PHY check");
4608 }
4609 }
4610 }
4611
4612 /*
4613 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4614 */
4615 static uint_t
4616 ixgbe_intr_legacy(void *arg1, void *arg2)
4617 {
4618 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4619 struct ixgbe_hw *hw = &ixgbe->hw;
4620 ixgbe_tx_ring_t *tx_ring;
4621 ixgbe_rx_ring_t *rx_ring;
4622 uint32_t eicr;
4623 mblk_t *mp;
4624 boolean_t tx_reschedule;
4625 uint_t result;
4626
4627 _NOTE(ARGUNUSED(arg2));
4628
4629 mutex_enter(&ixgbe->gen_lock);
4630 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4631 mutex_exit(&ixgbe->gen_lock);
4632 return (DDI_INTR_UNCLAIMED);
4633 }
4634
4635 mp = NULL;
4636 tx_reschedule = B_FALSE;
4637
4638 /*
4639 * Any bit set in eicr: claim this interrupt
4640 */
4641 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4642
4643 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4644 mutex_exit(&ixgbe->gen_lock);
4645 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4646 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4647 return (DDI_INTR_CLAIMED);
4648 }
4649
4650 if (eicr) {
4651 /*
4652 * For legacy interrupt, we have only one interrupt,
4653 * so we have only one rx ring and one tx ring enabled.
4654 */
4655 ASSERT(ixgbe->num_rx_rings == 1);
4656 ASSERT(ixgbe->num_tx_rings == 1);
4657
4658 /*
4659 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4660 */
4661 if (eicr & 0x1) {
4662 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4663 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4664 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4665 /*
4666 * Clean the rx descriptors
4667 */
4668 rx_ring = &ixgbe->rx_rings[0];
4669 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4670 }
4671
4672 /*
4673 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4674 */
4675 if (eicr & 0x2) {
4676 /*
4677 * Recycle the tx descriptors
4678 */
4679 tx_ring = &ixgbe->tx_rings[0];
4680 tx_ring->tx_recycle(tx_ring);
4681
4682 /*
4683 * Schedule the re-transmit
4684 */
4685 tx_reschedule = (tx_ring->reschedule &&
4686 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4687 }
4688
4689 /* any interrupt type other than tx/rx */
4690 if (eicr & ixgbe->capab->other_intr) {
4691 switch (hw->mac.type) {
4692 case ixgbe_mac_82598EB:
4693 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4694 break;
4695
4696 case ixgbe_mac_82599EB:
4697 case ixgbe_mac_X540:
4698 case ixgbe_mac_X550:
4699 case ixgbe_mac_X550EM_x:
4700 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4701 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4702 break;
4703
4704 default:
4705 break;
4706 }
4707 ixgbe_intr_other_work(ixgbe, eicr);
4708 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4709 }
4710
4711 mutex_exit(&ixgbe->gen_lock);
4712
4713 result = DDI_INTR_CLAIMED;
4714 } else {
4715 mutex_exit(&ixgbe->gen_lock);
4716
4717 /*
4718 * No interrupt cause bits set: don't claim this interrupt.
4719 */
4720 result = DDI_INTR_UNCLAIMED;
4721 }
4722
4723 /* re-enable the interrupts which were automasked */
4724 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4725
4726 /*
4727 * Do the following work outside of the gen_lock
4728 */
4729 if (mp != NULL) {
4730 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4731 rx_ring->ring_gen_num);
4732 }
4733
4734 if (tx_reschedule) {
4735 tx_ring->reschedule = B_FALSE;
4736 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4737 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4738 }
4739
4740 return (result);
4741 }
4742
4743 /*
4744 * ixgbe_intr_msi - Interrupt handler for MSI.
4745 */
4746 static uint_t
4747 ixgbe_intr_msi(void *arg1, void *arg2)
4748 {
4749 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4750 struct ixgbe_hw *hw = &ixgbe->hw;
4751 uint32_t eicr;
4752
4753 _NOTE(ARGUNUSED(arg2));
4754
4755 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4756
4757 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4758 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4759 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4760 return (DDI_INTR_CLAIMED);
4761 }
4762
4763 /*
4764 * For MSI interrupt, we have only one vector,
4765 * so we have only one rx ring and one tx ring enabled.
4766 */
4767 ASSERT(ixgbe->num_rx_rings == 1);
4768 ASSERT(ixgbe->num_tx_rings == 1);
4769
4770 /*
4771 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4772 */
4773 if (eicr & 0x1) {
4774 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4775 }
4776
4777 /*
4778 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4779 */
4780 if (eicr & 0x2) {
4781 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4782 }
4783
4784 /* any interrupt type other than tx/rx */
4785 if (eicr & ixgbe->capab->other_intr) {
4786 mutex_enter(&ixgbe->gen_lock);
4787 switch (hw->mac.type) {
4788 case ixgbe_mac_82598EB:
4789 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4790 break;
4791
4792 case ixgbe_mac_82599EB:
4793 case ixgbe_mac_X540:
4794 case ixgbe_mac_X550:
4795 case ixgbe_mac_X550EM_x:
4796 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4797 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4798 break;
4799
4800 default:
4801 break;
4802 }
4803 ixgbe_intr_other_work(ixgbe, eicr);
4804 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4805 mutex_exit(&ixgbe->gen_lock);
4806 }
4807
4808 /* re-enable the interrupts which were automasked */
4809 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4810
4811 return (DDI_INTR_CLAIMED);
4812 }
4813
4814 /*
4815 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4816 */
4817 static uint_t
4818 ixgbe_intr_msix(void *arg1, void *arg2)
4819 {
4820 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4821 ixgbe_t *ixgbe = vect->ixgbe;
4822 struct ixgbe_hw *hw = &ixgbe->hw;
4823 uint32_t eicr;
4824 int r_idx = 0;
4825
4826 _NOTE(ARGUNUSED(arg2));
4827
4828 /*
4829 * Clean each rx ring that has its bit set in the map
4830 */
4831 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4832 while (r_idx >= 0) {
4833 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4834 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4835 (ixgbe->num_rx_rings - 1));
4836 }
4837
4838 /*
4839 * Clean each tx ring that has its bit set in the map
4840 */
4841 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4842 while (r_idx >= 0) {
4843 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4844 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4845 (ixgbe->num_tx_rings - 1));
4846 }
4847
4848
4849 /*
4850 * Clean other interrupt (link change) that has its bit set in the map
4851 */
4852 if (BT_TEST(vect->other_map, 0) == 1) {
4853 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4854
4855 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4856 DDI_FM_OK) {
4857 ddi_fm_service_impact(ixgbe->dip,
4858 DDI_SERVICE_DEGRADED);
4859 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4860 return (DDI_INTR_CLAIMED);
4861 }
4862
4863 /*
4864 * Check "other" cause bits: any interrupt type other than tx/rx
4865 */
4866 if (eicr & ixgbe->capab->other_intr) {
4867 mutex_enter(&ixgbe->gen_lock);
4868 switch (hw->mac.type) {
4869 case ixgbe_mac_82598EB:
4870 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4871 ixgbe_intr_other_work(ixgbe, eicr);
4872 break;
4873
4874 case ixgbe_mac_82599EB:
4875 case ixgbe_mac_X540:
4876 case ixgbe_mac_X550:
4877 case ixgbe_mac_X550EM_x:
4878 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4879 ixgbe_intr_other_work(ixgbe, eicr);
4880 break;
4881
4882 default:
4883 break;
4884 }
4885 mutex_exit(&ixgbe->gen_lock);
4886 }
4887
4888 /* re-enable the interrupts which were automasked */
4889 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4890 }
4891
4892 return (DDI_INTR_CLAIMED);
4893 }
4894
4895 /*
4896 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4897 *
4898 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4899 * if not successful, try Legacy.
4900 * ixgbe->intr_force can be used to force sequence to start with
4901 * any of the 3 types.
4902 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4903 */
4904 static int
4905 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4906 {
4907 dev_info_t *devinfo;
4908 int intr_types;
4909 int rc;
4910
4911 devinfo = ixgbe->dip;
4912
4913 /*
4914 * Get supported interrupt types
4915 */
4916 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4917
4918 if (rc != DDI_SUCCESS) {
4919 ixgbe_log(ixgbe,
4920 "Get supported interrupt types failed: %d", rc);
4921 return (IXGBE_FAILURE);
4922 }
4923 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4924
4925 ixgbe->intr_type = 0;
4926
4927 /*
4928 * Install MSI-X interrupts
4929 */
4930 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4931 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4932 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4933 if (rc == IXGBE_SUCCESS)
4934 return (IXGBE_SUCCESS);
4935
4936 ixgbe_log(ixgbe,
4937 "Allocate MSI-X failed, trying MSI interrupts...");
4938 }
4939
4940 /*
4941 * MSI-X not used, force rings and groups to 1
4942 */
4943 ixgbe->num_rx_rings = 1;
4944 ixgbe->num_rx_groups = 1;
4945 ixgbe->num_tx_rings = 1;
4946 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4947 ixgbe_log(ixgbe,
4948 "MSI-X not used, force rings and groups number to 1");
4949
4950 /*
4951 * Install MSI interrupts
4952 */
4953 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4954 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4955 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4956 if (rc == IXGBE_SUCCESS)
4957 return (IXGBE_SUCCESS);
4958
4959 ixgbe_log(ixgbe,
4960 "Allocate MSI failed, trying Legacy interrupts...");
4961 }
4962
4963 /*
4964 * Install legacy interrupts
4965 */
4966 if (intr_types & DDI_INTR_TYPE_FIXED) {
4967 /*
4968 * Disallow legacy interrupts for X550. X550 has a silicon
4969 * bug which prevents Shared Legacy interrupts from working.
4970 * For details, please reference:
4971 *
4972 * Intel Ethernet Controller X550 Specification Update rev. 2.1
4973 * May 2016, erratum 22: PCIe Interrupt Status Bit
4974 */
4975 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
4976 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
4977 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
4978 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf) {
4979 ixgbe_log(ixgbe,
4980 "Legacy interrupts are not supported on this "
4981 "adapter. Please use MSI or MSI-X instead.");
4982 return (IXGBE_FAILURE);
4983 }
4984 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4985 if (rc == IXGBE_SUCCESS)
4986 return (IXGBE_SUCCESS);
4987
4988 ixgbe_log(ixgbe,
4989 "Allocate Legacy interrupts failed");
4990 }
4991
4992 /*
4993 * If none of the 3 types succeeded, return failure
4994 */
4995 return (IXGBE_FAILURE);
4996 }
4997
4998 /*
4999 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5000 *
5001 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5002 * if fewer than 2 handles are available, return failure.
5003 * Upon success, this maps the vectors to rx and tx rings for
5004 * interrupts.
5005 */
5006 static int
5007 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5008 {
5009 dev_info_t *devinfo;
5010 int request, count, actual;
5011 int minimum;
5012 int rc;
5013 uint32_t ring_per_group;
5014
5015 devinfo = ixgbe->dip;
5016
5017 switch (intr_type) {
5018 case DDI_INTR_TYPE_FIXED:
5019 request = 1; /* Request 1 legacy interrupt handle */
5020 minimum = 1;
5021 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5022 break;
5023
5024 case DDI_INTR_TYPE_MSI:
5025 request = 1; /* Request 1 MSI interrupt handle */
5026 minimum = 1;
5027 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5028 break;
5029
5030 case DDI_INTR_TYPE_MSIX:
5031 /*
5032 * Best number of vectors for the adapter is
5033 * (# rx rings + # tx rings), however we will
5034 * limit the request number.
5035 */
5036 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5037 if (request > ixgbe->capab->max_ring_vect)
5038 request = ixgbe->capab->max_ring_vect;
5039 minimum = 1;
5040 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5041 break;
5042
5043 default:
5044 ixgbe_log(ixgbe,
5045 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5046 intr_type);
5047 return (IXGBE_FAILURE);
5048 }
5049 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5050 request, minimum);
5051
5052 /*
5053 * Get number of supported interrupts
5054 */
5055 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5056 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5057 ixgbe_log(ixgbe,
5058 "Get interrupt number failed. Return: %d, count: %d",
5059 rc, count);
5060 return (IXGBE_FAILURE);
5061 }
5062 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5063
5064 actual = 0;
5065 ixgbe->intr_cnt = 0;
5066 ixgbe->intr_cnt_max = 0;
5067 ixgbe->intr_cnt_min = 0;
5068
5069 /*
5070 * Allocate an array of interrupt handles
5071 */
5072 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5073 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5074
5075 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5076 request, &actual, DDI_INTR_ALLOC_NORMAL);
5077 if (rc != DDI_SUCCESS) {
5078 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5079 "return: %d, request: %d, actual: %d",
5080 rc, request, actual);
5081 goto alloc_handle_fail;
5082 }
5083 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5084
5085 /*
5086 * upper/lower limit of interrupts
5087 */
5088 ixgbe->intr_cnt = actual;
5089 ixgbe->intr_cnt_max = request;
5090 ixgbe->intr_cnt_min = minimum;
5091
5092 /*
5093 * rss number per group should not exceed the rx interrupt number,
5094 * else need to adjust rx ring number.
5095 */
5096 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5097 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5098 if (actual < ring_per_group) {
5099 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5100 ixgbe_setup_vmdq_rss_conf(ixgbe);
5101 }
5102
5103 /*
5104 * Now we know the actual number of vectors. Here we map the vector
5105 * to other, rx rings and tx ring.
5106 */
5107 if (actual < minimum) {
5108 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5109 actual);
5110 goto alloc_handle_fail;
5111 }
5112
5113 /*
5114 * Get priority for first vector, assume remaining are all the same
5115 */
5116 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5117 if (rc != DDI_SUCCESS) {
5118 ixgbe_log(ixgbe,
5119 "Get interrupt priority failed: %d", rc);
5120 goto alloc_handle_fail;
5121 }
5122
5123 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5124 if (rc != DDI_SUCCESS) {
5125 ixgbe_log(ixgbe,
5126 "Get interrupt cap failed: %d", rc);
5127 goto alloc_handle_fail;
5128 }
5129
5130 ixgbe->intr_type = intr_type;
5131
5132 return (IXGBE_SUCCESS);
5133
5134 alloc_handle_fail:
5135 ixgbe_rem_intrs(ixgbe);
5136
5137 return (IXGBE_FAILURE);
5138 }
5139
5140 /*
5141 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5142 *
5143 * Before adding the interrupt handlers, the interrupt vectors have
5144 * been allocated, and the rx/tx rings have also been allocated.
5145 */
5146 static int
5147 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5148 {
5149 int vector = 0;
5150 int rc;
5151
5152 switch (ixgbe->intr_type) {
5153 case DDI_INTR_TYPE_MSIX:
5154 /*
5155 * Add interrupt handler for all vectors
5156 */
5157 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5158 /*
5159 * install pointer to vect_map[vector]
5160 */
5161 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5162 (ddi_intr_handler_t *)ixgbe_intr_msix,
5163 (void *)&ixgbe->vect_map[vector], NULL);
5164
5165 if (rc != DDI_SUCCESS) {
5166 ixgbe_log(ixgbe,
5167 "Add interrupt handler failed. "
5168 "return: %d, vector: %d", rc, vector);
5169 for (vector--; vector >= 0; vector--) {
5170 (void) ddi_intr_remove_handler(
5171 ixgbe->htable[vector]);
5172 }
5173 return (IXGBE_FAILURE);
5174 }
5175 }
5176
5177 break;
5178
5179 case DDI_INTR_TYPE_MSI:
5180 /*
5181 * Add interrupt handlers for the only vector
5182 */
5183 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5184 (ddi_intr_handler_t *)ixgbe_intr_msi,
5185 (void *)ixgbe, NULL);
5186
5187 if (rc != DDI_SUCCESS) {
5188 ixgbe_log(ixgbe,
5189 "Add MSI interrupt handler failed: %d", rc);
5190 return (IXGBE_FAILURE);
5191 }
5192
5193 break;
5194
5195 case DDI_INTR_TYPE_FIXED:
5196 /*
5197 * Add interrupt handlers for the only vector
5198 */
5199 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5200 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5201 (void *)ixgbe, NULL);
5202
5203 if (rc != DDI_SUCCESS) {
5204 ixgbe_log(ixgbe,
5205 "Add legacy interrupt handler failed: %d", rc);
5206 return (IXGBE_FAILURE);
5207 }
5208
5209 break;
5210
5211 default:
5212 return (IXGBE_FAILURE);
5213 }
5214
5215 return (IXGBE_SUCCESS);
5216 }
5217
5218 #pragma inline(ixgbe_map_rxring_to_vector)
5219 /*
5220 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5221 */
5222 static void
5223 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5224 {
5225 /*
5226 * Set bit in map
5227 */
5228 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5229
5230 /*
5231 * Count bits set
5232 */
5233 ixgbe->vect_map[v_idx].rxr_cnt++;
5234
5235 /*
5236 * Remember bit position
5237 */
5238 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5239 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5240 }
5241
5242 #pragma inline(ixgbe_map_txring_to_vector)
5243 /*
5244 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5245 */
5246 static void
5247 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5248 {
5249 /*
5250 * Set bit in map
5251 */
5252 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5253
5254 /*
5255 * Count bits set
5256 */
5257 ixgbe->vect_map[v_idx].txr_cnt++;
5258
5259 /*
5260 * Remember bit position
5261 */
5262 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5263 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5264 }
5265
5266 /*
5267 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5268 * allocation register (IVAR).
5269 * cause:
5270 * -1 : other cause
5271 * 0 : rx
5272 * 1 : tx
5273 */
5274 static void
5275 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5276 int8_t cause)
5277 {
5278 struct ixgbe_hw *hw = &ixgbe->hw;
5279 u32 ivar, index;
5280
5281 switch (hw->mac.type) {
5282 case ixgbe_mac_82598EB:
5283 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5284 if (cause == -1) {
5285 cause = 0;
5286 }
5287 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5288 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5289 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5290 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5291 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5292 break;
5293
5294 case ixgbe_mac_82599EB:
5295 case ixgbe_mac_X540:
5296 case ixgbe_mac_X550:
5297 case ixgbe_mac_X550EM_x:
5298 if (cause == -1) {
5299 /* other causes */
5300 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5301 index = (intr_alloc_entry & 1) * 8;
5302 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5303 ivar &= ~(0xFF << index);
5304 ivar |= (msix_vector << index);
5305 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5306 } else {
5307 /* tx or rx causes */
5308 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5309 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5310 ivar = IXGBE_READ_REG(hw,
5311 IXGBE_IVAR(intr_alloc_entry >> 1));
5312 ivar &= ~(0xFF << index);
5313 ivar |= (msix_vector << index);
5314 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5315 ivar);
5316 }
5317 break;
5318
5319 default:
5320 break;
5321 }
5322 }
5323
5324 /*
5325 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5326 * given interrupt vector allocation register (IVAR).
5327 * cause:
5328 * -1 : other cause
5329 * 0 : rx
5330 * 1 : tx
5331 */
5332 static void
5333 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5334 {
5335 struct ixgbe_hw *hw = &ixgbe->hw;
5336 u32 ivar, index;
5337
5338 switch (hw->mac.type) {
5339 case ixgbe_mac_82598EB:
5340 if (cause == -1) {
5341 cause = 0;
5342 }
5343 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5344 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5345 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5346 (intr_alloc_entry & 0x3)));
5347 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5348 break;
5349
5350 case ixgbe_mac_82599EB:
5351 case ixgbe_mac_X540:
5352 case ixgbe_mac_X550:
5353 case ixgbe_mac_X550EM_x:
5354 if (cause == -1) {
5355 /* other causes */
5356 index = (intr_alloc_entry & 1) * 8;
5357 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5358 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5359 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5360 } else {
5361 /* tx or rx causes */
5362 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5363 ivar = IXGBE_READ_REG(hw,
5364 IXGBE_IVAR(intr_alloc_entry >> 1));
5365 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5366 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5367 ivar);
5368 }
5369 break;
5370
5371 default:
5372 break;
5373 }
5374 }
5375
5376 /*
5377 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5378 * given interrupt vector allocation register (IVAR).
5379 * cause:
5380 * -1 : other cause
5381 * 0 : rx
5382 * 1 : tx
5383 */
5384 static void
5385 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5386 {
5387 struct ixgbe_hw *hw = &ixgbe->hw;
5388 u32 ivar, index;
5389
5390 switch (hw->mac.type) {
5391 case ixgbe_mac_82598EB:
5392 if (cause == -1) {
5393 cause = 0;
5394 }
5395 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5396 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5397 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5398 (intr_alloc_entry & 0x3)));
5399 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5400 break;
5401
5402 case ixgbe_mac_82599EB:
5403 case ixgbe_mac_X540:
5404 case ixgbe_mac_X550:
5405 case ixgbe_mac_X550EM_x:
5406 if (cause == -1) {
5407 /* other causes */
5408 index = (intr_alloc_entry & 1) * 8;
5409 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5410 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5411 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5412 } else {
5413 /* tx or rx causes */
5414 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5415 ivar = IXGBE_READ_REG(hw,
5416 IXGBE_IVAR(intr_alloc_entry >> 1));
5417 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5418 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5419 ivar);
5420 }
5421 break;
5422
5423 default:
5424 break;
5425 }
5426 }
5427
5428 /*
5429 * Convert the rx ring index driver maintained to the rx ring index
5430 * in h/w.
5431 */
5432 static uint32_t
5433 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5434 {
5435
5436 struct ixgbe_hw *hw = &ixgbe->hw;
5437 uint32_t rx_ring_per_group, hw_rx_index;
5438
5439 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5440 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5441 return (sw_rx_index);
5442 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5443 switch (hw->mac.type) {
5444 case ixgbe_mac_82598EB:
5445 return (sw_rx_index);
5446
5447 case ixgbe_mac_82599EB:
5448 case ixgbe_mac_X540:
5449 case ixgbe_mac_X550:
5450 case ixgbe_mac_X550EM_x:
5451 return (sw_rx_index * 2);
5452
5453 default:
5454 break;
5455 }
5456 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5457 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5458
5459 switch (hw->mac.type) {
5460 case ixgbe_mac_82598EB:
5461 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5462 16 + (sw_rx_index % rx_ring_per_group);
5463 return (hw_rx_index);
5464
5465 case ixgbe_mac_82599EB:
5466 case ixgbe_mac_X540:
5467 case ixgbe_mac_X550:
5468 case ixgbe_mac_X550EM_x:
5469 if (ixgbe->num_rx_groups > 32) {
5470 hw_rx_index = (sw_rx_index /
5471 rx_ring_per_group) * 2 +
5472 (sw_rx_index % rx_ring_per_group);
5473 } else {
5474 hw_rx_index = (sw_rx_index /
5475 rx_ring_per_group) * 4 +
5476 (sw_rx_index % rx_ring_per_group);
5477 }
5478 return (hw_rx_index);
5479
5480 default:
5481 break;
5482 }
5483 }
5484
5485 /*
5486 * Should never reach. Just to make compiler happy.
5487 */
5488 return (sw_rx_index);
5489 }
5490
5491 /*
5492 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5493 *
5494 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5495 * to vector[0 - (intr_cnt -1)].
5496 */
5497 static int
5498 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5499 {
5500 int i, vector = 0;
5501
5502 /* initialize vector map */
5503 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5504 for (i = 0; i < ixgbe->intr_cnt; i++) {
5505 ixgbe->vect_map[i].ixgbe = ixgbe;
5506 }
5507
5508 /*
5509 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5510 * tx rings[0] on RTxQ[1].
5511 */
5512 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5513 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5514 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5515 return (IXGBE_SUCCESS);
5516 }
5517
5518 /*
5519 * Interrupts/vectors mapping for MSI-X
5520 */
5521
5522 /*
5523 * Map other interrupt to vector 0,
5524 * Set bit in map and count the bits set.
5525 */
5526 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5527 ixgbe->vect_map[vector].other_cnt++;
5528
5529 /*
5530 * Map rx ring interrupts to vectors
5531 */
5532 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5533 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5534 vector = (vector +1) % ixgbe->intr_cnt;
5535 }
5536
5537 /*
5538 * Map tx ring interrupts to vectors
5539 */
5540 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5541 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5542 vector = (vector +1) % ixgbe->intr_cnt;
5543 }
5544
5545 return (IXGBE_SUCCESS);
5546 }
5547
5548 /*
5549 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5550 *
5551 * This relies on ring/vector mapping already set up in the
5552 * vect_map[] structures
5553 */
5554 static void
5555 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5556 {
5557 struct ixgbe_hw *hw = &ixgbe->hw;
5558 ixgbe_intr_vector_t *vect; /* vector bitmap */
5559 int r_idx; /* ring index */
5560 int v_idx; /* vector index */
5561 uint32_t hw_index;
5562
5563 /*
5564 * Clear any previous entries
5565 */
5566 switch (hw->mac.type) {
5567 case ixgbe_mac_82598EB:
5568 for (v_idx = 0; v_idx < 25; v_idx++)
5569 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5570 break;
5571
5572 case ixgbe_mac_82599EB:
5573 case ixgbe_mac_X540:
5574 case ixgbe_mac_X550:
5575 case ixgbe_mac_X550EM_x:
5576 for (v_idx = 0; v_idx < 64; v_idx++)
5577 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5578 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5579 break;
5580
5581 default:
5582 break;
5583 }
5584
5585 /*
5586 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5587 * tx rings[0] will use RTxQ[1].
5588 */
5589 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5590 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5591 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5592 return;
5593 }
5594
5595 /*
5596 * For MSI-X interrupt, "Other" is always on vector[0].
5597 */
5598 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5599
5600 /*
5601 * For each interrupt vector, populate the IVAR table
5602 */
5603 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5604 vect = &ixgbe->vect_map[v_idx];
5605
5606 /*
5607 * For each rx ring bit set
5608 */
5609 r_idx = bt_getlowbit(vect->rx_map, 0,
5610 (ixgbe->num_rx_rings - 1));
5611
5612 while (r_idx >= 0) {
5613 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5614 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5615 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5616 (ixgbe->num_rx_rings - 1));
5617 }
5618
5619 /*
5620 * For each tx ring bit set
5621 */
5622 r_idx = bt_getlowbit(vect->tx_map, 0,
5623 (ixgbe->num_tx_rings - 1));
5624
5625 while (r_idx >= 0) {
5626 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5627 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5628 (ixgbe->num_tx_rings - 1));
5629 }
5630 }
5631 }
5632
5633 /*
5634 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5635 */
5636 static void
5637 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5638 {
5639 int i;
5640 int rc;
5641
5642 for (i = 0; i < ixgbe->intr_cnt; i++) {
5643 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5644 if (rc != DDI_SUCCESS) {
5645 IXGBE_DEBUGLOG_1(ixgbe,
5646 "Remove intr handler failed: %d", rc);
5647 }
5648 }
5649 }
5650
5651 /*
5652 * ixgbe_rem_intrs - Remove the allocated interrupts.
5653 */
5654 static void
5655 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5656 {
5657 int i;
5658 int rc;
5659
5660 for (i = 0; i < ixgbe->intr_cnt; i++) {
5661 rc = ddi_intr_free(ixgbe->htable[i]);
5662 if (rc != DDI_SUCCESS) {
5663 IXGBE_DEBUGLOG_1(ixgbe,
5664 "Free intr failed: %d", rc);
5665 }
5666 }
5667
5668 kmem_free(ixgbe->htable, ixgbe->intr_size);
5669 ixgbe->htable = NULL;
5670 }
5671
5672 /*
5673 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5674 */
5675 static int
5676 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5677 {
5678 int i;
5679 int rc;
5680
5681 /*
5682 * Enable interrupts
5683 */
5684 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5685 /*
5686 * Call ddi_intr_block_enable() for MSI
5687 */
5688 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5689 if (rc != DDI_SUCCESS) {
5690 ixgbe_log(ixgbe,
5691 "Enable block intr failed: %d", rc);
5692 return (IXGBE_FAILURE);
5693 }
5694 } else {
5695 /*
5696 * Call ddi_intr_enable() for Legacy/MSI non block enable
5697 */
5698 for (i = 0; i < ixgbe->intr_cnt; i++) {
5699 rc = ddi_intr_enable(ixgbe->htable[i]);
5700 if (rc != DDI_SUCCESS) {
5701 ixgbe_log(ixgbe,
5702 "Enable intr failed: %d", rc);
5703 return (IXGBE_FAILURE);
5704 }
5705 }
5706 }
5707
5708 return (IXGBE_SUCCESS);
5709 }
5710
5711 /*
5712 * ixgbe_disable_intrs - Disable all the interrupts.
5713 */
5714 static int
5715 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5716 {
5717 int i;
5718 int rc;
5719
5720 /*
5721 * Disable all interrupts
5722 */
5723 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5724 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5725 if (rc != DDI_SUCCESS) {
5726 ixgbe_log(ixgbe,
5727 "Disable block intr failed: %d", rc);
5728 return (IXGBE_FAILURE);
5729 }
5730 } else {
5731 for (i = 0; i < ixgbe->intr_cnt; i++) {
5732 rc = ddi_intr_disable(ixgbe->htable[i]);
5733 if (rc != DDI_SUCCESS) {
5734 ixgbe_log(ixgbe,
5735 "Disable intr failed: %d", rc);
5736 return (IXGBE_FAILURE);
5737 }
5738 }
5739 }
5740
5741 return (IXGBE_SUCCESS);
5742 }
5743
5744 /*
5745 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5746 */
5747 static void
5748 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5749 {
5750 struct ixgbe_hw *hw = &ixgbe->hw;
5751 ixgbe_link_speed speed = 0;
5752 boolean_t link_up = B_FALSE;
5753 uint32_t pcs1g_anlp = 0;
5754
5755 ASSERT(mutex_owned(&ixgbe->gen_lock));
5756 ixgbe->param_lp_1000fdx_cap = 0;
5757 ixgbe->param_lp_100fdx_cap = 0;
5758
5759 /* check for link, don't wait */
5760 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5761
5762 /*
5763 * Update the observed Link Partner's capabilities. Not all adapters
5764 * can provide full information on the LP's capable speeds, so we
5765 * provide what we can.
5766 */
5767 if (link_up) {
5768 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5769
5770 ixgbe->param_lp_1000fdx_cap =
5771 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5772 ixgbe->param_lp_100fdx_cap =
5773 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5774 }
5775
5776 /*
5777 * Update GLD's notion of the adapter's currently advertised speeds.
5778 * Since the common code doesn't always record the current autonegotiate
5779 * settings in the phy struct for all parts (specifically, adapters with
5780 * SFPs) we first test to see if it is 0, and if so, we fall back to
5781 * using the adapter's speed capabilities which we saved during instance
5782 * init in ixgbe_init_params().
5783 *
5784 * Adapters with SFPs will always be shown as advertising all of their
5785 * supported speeds, and adapters with baseT PHYs (where the phy struct
5786 * is maintained by the common code) will always have a factual view of
5787 * their currently-advertised speeds. In the case of SFPs, this is
5788 * acceptable as we default to advertising all speeds that the adapter
5789 * claims to support, and those properties are immutable; unlike on
5790 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5791 */
5792 speed = hw->phy.autoneg_advertised;
5793 if (speed == 0)
5794 speed = ixgbe->speeds_supported;
5795
5796 ixgbe->param_adv_10000fdx_cap =
5797 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5798 ixgbe->param_adv_5000fdx_cap =
5799 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5800 ixgbe->param_adv_2500fdx_cap =
5801 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5802 ixgbe->param_adv_1000fdx_cap =
5803 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5804 ixgbe->param_adv_100fdx_cap =
5805 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5806 }
5807
5808 /*
5809 * ixgbe_get_driver_control - Notify that driver is in control of device.
5810 */
5811 static void
5812 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5813 {
5814 uint32_t ctrl_ext;
5815
5816 /*
5817 * Notify firmware that driver is in control of device
5818 */
5819 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5820 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5821 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5822 }
5823
5824 /*
5825 * ixgbe_release_driver_control - Notify that driver is no longer in control
5826 * of device.
5827 */
5828 static void
5829 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5830 {
5831 uint32_t ctrl_ext;
5832
5833 /*
5834 * Notify firmware that driver is no longer in control of device
5835 */
5836 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5837 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5838 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5839 }
5840
5841 /*
5842 * ixgbe_atomic_reserve - Atomic decrease operation.
5843 */
5844 int
5845 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5846 {
5847 uint32_t oldval;
5848 uint32_t newval;
5849
5850 /*
5851 * ATOMICALLY
5852 */
5853 do {
5854 oldval = *count_p;
5855 if (oldval < n)
5856 return (-1);
5857 newval = oldval - n;
5858 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5859
5860 return (newval);
5861 }
5862
5863 /*
5864 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5865 */
5866 static uint8_t *
5867 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5868 {
5869 uint8_t *addr = *upd_ptr;
5870 uint8_t *new_ptr;
5871
5872 _NOTE(ARGUNUSED(hw));
5873 _NOTE(ARGUNUSED(vmdq));
5874
5875 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5876 *upd_ptr = new_ptr;
5877 return (addr);
5878 }
5879
5880 /*
5881 * FMA support
5882 */
5883 int
5884 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5885 {
5886 ddi_fm_error_t de;
5887
5888 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5889 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5890 return (de.fme_status);
5891 }
5892
5893 int
5894 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5895 {
5896 ddi_fm_error_t de;
5897
5898 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5899 return (de.fme_status);
5900 }
5901
5902 /*
5903 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5904 */
5905 static int
5906 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5907 {
5908 _NOTE(ARGUNUSED(impl_data));
5909 /*
5910 * as the driver can always deal with an error in any dma or
5911 * access handle, we can just return the fme_status value.
5912 */
5913 pci_ereport_post(dip, err, NULL);
5914 return (err->fme_status);
5915 }
5916
5917 static void
5918 ixgbe_fm_init(ixgbe_t *ixgbe)
5919 {
5920 ddi_iblock_cookie_t iblk;
5921 int fma_dma_flag;
5922
5923 /*
5924 * Only register with IO Fault Services if we have some capability
5925 */
5926 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5927 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5928 } else {
5929 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5930 }
5931
5932 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5933 fma_dma_flag = 1;
5934 } else {
5935 fma_dma_flag = 0;
5936 }
5937
5938 ixgbe_set_fma_flags(fma_dma_flag);
5939
5940 if (ixgbe->fm_capabilities) {
5941
5942 /*
5943 * Register capabilities with IO Fault Services
5944 */
5945 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5946
5947 /*
5948 * Initialize pci ereport capabilities if ereport capable
5949 */
5950 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5951 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5952 pci_ereport_setup(ixgbe->dip);
5953
5954 /*
5955 * Register error callback if error callback capable
5956 */
5957 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5958 ddi_fm_handler_register(ixgbe->dip,
5959 ixgbe_fm_error_cb, (void*) ixgbe);
5960 }
5961 }
5962
5963 static void
5964 ixgbe_fm_fini(ixgbe_t *ixgbe)
5965 {
5966 /*
5967 * Only unregister FMA capabilities if they are registered
5968 */
5969 if (ixgbe->fm_capabilities) {
5970
5971 /*
5972 * Release any resources allocated by pci_ereport_setup()
5973 */
5974 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5975 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5976 pci_ereport_teardown(ixgbe->dip);
5977
5978 /*
5979 * Un-register error callback if error callback capable
5980 */
5981 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5982 ddi_fm_handler_unregister(ixgbe->dip);
5983
5984 /*
5985 * Unregister from IO Fault Service
5986 */
5987 ddi_fm_fini(ixgbe->dip);
5988 }
5989 }
5990
5991 void
5992 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5993 {
5994 uint64_t ena;
5995 char buf[FM_MAX_CLASS];
5996
5997 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5998 ena = fm_ena_generate(0, FM_ENA_FMT1);
5999 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6000 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6001 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6002 }
6003 }
6004
6005 static int
6006 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6007 {
6008 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6009
6010 mutex_enter(&rx_ring->rx_lock);
6011 rx_ring->ring_gen_num = mr_gen_num;
6012 mutex_exit(&rx_ring->rx_lock);
6013 return (0);
6014 }
6015
6016 /*
6017 * Get the global ring index by a ring index within a group.
6018 */
6019 static int
6020 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6021 {
6022 ixgbe_rx_ring_t *rx_ring;
6023 int i;
6024
6025 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6026 rx_ring = &ixgbe->rx_rings[i];
6027 if (rx_ring->group_index == gindex)
6028 rindex--;
6029 if (rindex < 0)
6030 return (i);
6031 }
6032
6033 return (-1);
6034 }
6035
6036 /*
6037 * Callback funtion for MAC layer to register all rings.
6038 */
6039 /* ARGSUSED */
6040 void
6041 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6042 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6043 {
6044 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6045 mac_intr_t *mintr = &infop->mri_intr;
6046
6047 switch (rtype) {
6048 case MAC_RING_TYPE_RX: {
6049 /*
6050 * 'index' is the ring index within the group.
6051 * Need to get the global ring index by searching in groups.
6052 */
6053 int global_ring_index = ixgbe_get_rx_ring_index(
6054 ixgbe, group_index, ring_index);
6055
6056 ASSERT(global_ring_index >= 0);
6057
6058 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6059 rx_ring->ring_handle = rh;
6060
6061 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6062 infop->mri_start = ixgbe_ring_start;
6063 infop->mri_stop = NULL;
6064 infop->mri_poll = ixgbe_ring_rx_poll;
6065 infop->mri_stat = ixgbe_rx_ring_stat;
6066
6067 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6068 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6069 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6070 if (ixgbe->intr_type &
6071 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6072 mintr->mi_ddi_handle =
6073 ixgbe->htable[rx_ring->intr_vector];
6074 }
6075
6076 break;
6077 }
6078 case MAC_RING_TYPE_TX: {
6079 ASSERT(group_index == -1);
6080 ASSERT(ring_index < ixgbe->num_tx_rings);
6081
6082 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6083 tx_ring->ring_handle = rh;
6084
6085 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6086 infop->mri_start = NULL;
6087 infop->mri_stop = NULL;
6088 infop->mri_tx = ixgbe_ring_tx;
6089 infop->mri_stat = ixgbe_tx_ring_stat;
6090 if (ixgbe->intr_type &
6091 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6092 mintr->mi_ddi_handle =
6093 ixgbe->htable[tx_ring->intr_vector];
6094 }
6095 break;
6096 }
6097 default:
6098 break;
6099 }
6100 }
6101
6102 /*
6103 * Callback funtion for MAC layer to register all groups.
6104 */
6105 void
6106 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6107 mac_group_info_t *infop, mac_group_handle_t gh)
6108 {
6109 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6110
6111 switch (rtype) {
6112 case MAC_RING_TYPE_RX: {
6113 ixgbe_rx_group_t *rx_group;
6114
6115 rx_group = &ixgbe->rx_groups[index];
6116 rx_group->group_handle = gh;
6117
6118 infop->mgi_driver = (mac_group_driver_t)rx_group;
6119 infop->mgi_start = NULL;
6120 infop->mgi_stop = NULL;
6121 infop->mgi_addmac = ixgbe_addmac;
6122 infop->mgi_remmac = ixgbe_remmac;
6123 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6124
6125 break;
6126 }
6127 case MAC_RING_TYPE_TX:
6128 break;
6129 default:
6130 break;
6131 }
6132 }
6133
6134 /*
6135 * Enable interrupt on the specificed rx ring.
6136 */
6137 int
6138 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6139 {
6140 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6141 ixgbe_t *ixgbe = rx_ring->ixgbe;
6142 int r_idx = rx_ring->index;
6143 int hw_r_idx = rx_ring->hw_index;
6144 int v_idx = rx_ring->intr_vector;
6145
6146 mutex_enter(&ixgbe->gen_lock);
6147 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6148 mutex_exit(&ixgbe->gen_lock);
6149 /*
6150 * Simply return 0.
6151 * Interrupts are being adjusted. ixgbe_intr_adjust()
6152 * will eventually re-enable the interrupt when it's
6153 * done with the adjustment.
6154 */
6155 return (0);
6156 }
6157
6158 /*
6159 * To enable interrupt by setting the VAL bit of given interrupt
6160 * vector allocation register (IVAR).
6161 */
6162 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6163
6164 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6165
6166 /*
6167 * Trigger a Rx interrupt on this ring
6168 */
6169 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6170 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6171
6172 mutex_exit(&ixgbe->gen_lock);
6173
6174 return (0);
6175 }
6176
6177 /*
6178 * Disable interrupt on the specificed rx ring.
6179 */
6180 int
6181 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6182 {
6183 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6184 ixgbe_t *ixgbe = rx_ring->ixgbe;
6185 int r_idx = rx_ring->index;
6186 int hw_r_idx = rx_ring->hw_index;
6187 int v_idx = rx_ring->intr_vector;
6188
6189 mutex_enter(&ixgbe->gen_lock);
6190 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6191 mutex_exit(&ixgbe->gen_lock);
6192 /*
6193 * Simply return 0.
6194 * In the rare case where an interrupt is being
6195 * disabled while interrupts are being adjusted,
6196 * we don't fail the operation. No interrupts will
6197 * be generated while they are adjusted, and
6198 * ixgbe_intr_adjust() will cause the interrupts
6199 * to be re-enabled once it completes. Note that
6200 * in this case, packets may be delivered to the
6201 * stack via interrupts before xgbe_rx_ring_intr_enable()
6202 * is called again. This is acceptable since interrupt
6203 * adjustment is infrequent, and the stack will be
6204 * able to handle these packets.
6205 */
6206 return (0);
6207 }
6208
6209 /*
6210 * To disable interrupt by clearing the VAL bit of given interrupt
6211 * vector allocation register (IVAR).
6212 */
6213 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6214
6215 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6216
6217 mutex_exit(&ixgbe->gen_lock);
6218
6219 return (0);
6220 }
6221
6222 /*
6223 * Add a mac address.
6224 */
6225 static int
6226 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6227 {
6228 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6229 ixgbe_t *ixgbe = rx_group->ixgbe;
6230 struct ixgbe_hw *hw = &ixgbe->hw;
6231 int slot, i;
6232
6233 mutex_enter(&ixgbe->gen_lock);
6234
6235 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6236 mutex_exit(&ixgbe->gen_lock);
6237 return (ECANCELED);
6238 }
6239
6240 if (ixgbe->unicst_avail == 0) {
6241 /* no slots available */
6242 mutex_exit(&ixgbe->gen_lock);
6243 return (ENOSPC);
6244 }
6245
6246 /*
6247 * The first ixgbe->num_rx_groups slots are reserved for each respective
6248 * group. The rest slots are shared by all groups. While adding a
6249 * MAC address, reserved slots are firstly checked then the shared
6250 * slots are searched.
6251 */
6252 slot = -1;
6253 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6254 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6255 if (ixgbe->unicst_addr[i].mac.set == 0) {
6256 slot = i;
6257 break;
6258 }
6259 }
6260 } else {
6261 slot = rx_group->index;
6262 }
6263
6264 if (slot == -1) {
6265 /* no slots available */
6266 mutex_exit(&ixgbe->gen_lock);
6267 return (ENOSPC);
6268 }
6269
6270 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6271 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6272 rx_group->index, IXGBE_RAH_AV);
6273 ixgbe->unicst_addr[slot].mac.set = 1;
6274 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6275 ixgbe->unicst_avail--;
6276
6277 mutex_exit(&ixgbe->gen_lock);
6278
6279 return (0);
6280 }
6281
6282 /*
6283 * Remove a mac address.
6284 */
6285 static int
6286 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6287 {
6288 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6289 ixgbe_t *ixgbe = rx_group->ixgbe;
6290 struct ixgbe_hw *hw = &ixgbe->hw;
6291 int slot;
6292
6293 mutex_enter(&ixgbe->gen_lock);
6294
6295 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6296 mutex_exit(&ixgbe->gen_lock);
6297 return (ECANCELED);
6298 }
6299
6300 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6301 if (slot == -1) {
6302 mutex_exit(&ixgbe->gen_lock);
6303 return (EINVAL);
6304 }
6305
6306 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6307 mutex_exit(&ixgbe->gen_lock);
6308 return (EINVAL);
6309 }
6310
6311 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6312 (void) ixgbe_clear_rar(hw, slot);
6313 ixgbe->unicst_addr[slot].mac.set = 0;
6314 ixgbe->unicst_avail++;
6315
6316 mutex_exit(&ixgbe->gen_lock);
6317
6318 return (0);
6319 }