path: root/crates/imxrt/src/drivers/enet.rs

use crate::Instance;
use crate::ral::enet;
use core::{
    num::NonZeroU32,
    pin::Pin,
    sync::atomic::{self, Ordering},
};

use fusible::{
    event_flags::{EventFlags, EventFlagsContext, GetOption, SetOption},
    netx_duo::{
        driver,
        interface::{Capability, Interface},
        ip::Ip,
        packet::{self, PacketChainer, PacketList, PacketListChainView, PacketPool},
    },
};

use rust_threadx_net_phys::{Mdio, Phy};

use super::gpio::GpioOutput;

pub use enet::{RxRing, TxRing, rx_bd::RxBD, tx_bd::TxBD};

pub struct EnetContext {
    flags: EventFlagsContext<'static>,
}

const MII_EVENT: NonZeroU32 = NonZeroU32::new(enet::EIR::MII::mask).unwrap();

impl EnetContext {
    pub const fn new() -> Self {
        Self {
            flags: EventFlags::context(),
        }
    }

    #[inline(always)]
    pub unsafe fn on_interrupt(
        &'static self,
        enet: Instance<enet::RegisterBlock>,
        ip: &'static Ip,
    ) {
        unsafe {
            let eir = crate::read_reg!(enet, enet, EIR);
            crate::write_reg!(enet, enet, EIR, eir);

            if eir & DEFERRED_INTERRUPTS.get() != 0 {
                ip.deferred_processing();
            }

            Pin::static_ref(&self.flags)
                .assume_created()
                .set(eir, SetOption::Or);
        }
    }

    pub unsafe fn create<P: Phy<EnetMdio>>(
        &'static self,
        enet: Instance<enet::RegisterBlock>,
        tx_ring: &'static [TxBD],
        rx_ring: &'static [RxBD],
        phy: P,
        rst: GpioOutput,
        mdio_source_clock_hz: u32,
    ) -> Enet<P> {
        let flags = EventFlags::create(Pin::static_ref(&self.flags), &Default::default()).unwrap();

        crate::write_reg!(enet, enet, ECR, RESET: 1);

        const SMI_MDC_FREQUENCY_HZ: u32 = 2_500_000;
        let mii_speed = mdio_source_clock_hz.div_ceil(2 * SMI_MDC_FREQUENCY_HZ) - 1;
        let hold_time = 10_u32.div_ceil(1_000_000_000 / mdio_source_clock_hz) - 1;
        crate::modify_reg!(enet, enet, MSCR, HOLDTIME: hold_time, MII_SPEED: mii_speed);

        Enet {
            enet,
            phy,
            rst,
            flags,

            tx_ring: TransmitRing::new(tx_ring),
            rx_ring: ReceiveRing::new(rx_ring),
        }
    }
}

pub struct EnetMdio {
    enet: Instance<enet::RegisterBlock>,
    flags: &'static EventFlags,
}

pub struct Enet<P: Phy<EnetMdio>> {
    enet: Instance<enet::RegisterBlock>,
    phy: P,
    rst: GpioOutput,
    flags: &'static EventFlags,

    tx_ring: TransmitRing,
    rx_ring: ReceiveRing,
}

unsafe impl<P: Phy<EnetMdio>> Send for Enet<P> {}

impl Mdio for EnetMdio {
    type Error = core::convert::Infallible;

    fn read(&mut self, ctrl: u16) -> Result<u16, Self::Error> {
        let mmfr = (ctrl as u32) << 16;
        crate::write_reg!(enet, self.enet, MMFR, mmfr);

        self.flags.get(MII_EVENT, GetOption::AndClear).unwrap();

        // Automatically discards control bits.
        let data = crate::read_reg!(enet, self.enet, MMFR, DATA) as u16;
        Ok(data)
    }

    fn write(&mut self, ctrl: u16, data: u16) -> Result<(), Self::Error> {
        let mmfr = (ctrl as u32) << 16 | data as u32;
        crate::write_reg!(enet, self.enet, MMFR, mmfr);

        self.flags.get(MII_EVENT, GetOption::AndClear).unwrap();

        Ok(())
    }
}

impl<P: Phy<EnetMdio>> driver::Driver<'static> for Enet<P> {
    fn attach_interface(
        self: Pin<&'static mut Self>,
        mut extras: driver::DriverExtras<'static>,
    ) -> Result<(), driver::DriverError> {
        extras
            .interface
            .set_capability(
                extras.ip,
                Capability::IPV4_RX_CHECKSUM
                    | Capability::IPV4_TX_CHECKSUM
                    | Capability::UDP_RX_CHECKSUM
                    | Capability::UDP_TX_CHECKSUM
                    | Capability::TCP_RX_CHECKSUM
                    | Capability::TCP_TX_CHECKSUM,
            )
            .unwrap();

        // TODO: this may need to be bound by MRBR. Otherwise,
        // we're lying to the upper levels.
        extras.interface.set_mtu(extras.ip, 1500);
        extras.interface.driver_needs_mapping(extras.ip);

        Ok(())
    }

    fn initialize_link(
        self: Pin<&'static mut Self>,
        _: driver::DriverExtras<'static>,
    ) -> Result<(), driver::DriverError> {
        // Safety: not relocating anything in memory.
        let this = unsafe { self.get_unchecked_mut() };
        initialize_enet(
            this.enet,
            this.tx_ring.buffer_descriptors,
            this.rx_ring.buffer_descriptors,
            true,
        );

        this.rst.clear();
        fusible::thread::sleep(10);
        this.rst.set();
        fusible::thread::sleep(10);

        let mut mdio = EnetMdio {
            enet: this.enet.clone(),
            flags: this.flags,
        };

        this.phy.initialize(&mut mdio).unwrap();

        Ok(())
    }

    fn enable_link(
        self: Pin<&'static mut Self>,
        mut extras: driver::DriverExtras<'static>,
    ) -> Result<(), driver::DriverError> {
        // Safety: not moving out of self.
        let this = unsafe { self.get_unchecked_mut() };
        let mrbr = schedule_receives(&mut this.rx_ring, extras.ip.default_packet_pool());
        assert_ne!(mrbr, 0);

        crate::write_reg!(enet, this.enet, MRBR, mrbr as u32);
        crate::write_reg!(enet, this.enet, RDAR, RDAR: 1);

        extras.interface.set_link_up(true);

        Ok(())
    }

    fn set_physical_address(
        self: Pin<&'static mut Self>,
        extras: driver::DriverExtras<'static>,
    ) -> Result<(), driver::DriverError> {
        crate::write_reg!(enet, self.enet, PALR, extras.msw << 16 | (extras.lsw >> 16));
        crate::write_reg!(enet, self.enet, PAUR, extras.lsw << 16);
        Ok(())
    }

    fn deferred_processing(
        self: Pin<&'static mut Self>,
        extras: driver::DriverExtras<'static>,
    ) -> Result<(), driver::DriverError> {
        // Safety: not moving out of this object.
        let this = unsafe { self.get_unchecked_mut() };
        let eir = this.flags.try_get(DEFERRED_INTERRUPTS, GetOption::OrClear);
        let eir = eir.map_or(0, NonZeroU32::get);

        if eir & TRANSMIT_INTERRUPT_EVENTS != 0 {
            this.tx_ring.deallocate_completions();
        }

        if eir & RECEIVE_INTERRUPT_EVENTS != 0 {
            while let Some(packet) = this.rx_ring.try_frame_receive() {
                enqueue_rx_packet_to_ip(packet, extras.ip, extras.interface);
            }
        }

        // Make sure receive operations are primed. Otherwise,
        // we drop data. Once all descriptors are saturated,
        // try transmitting pending packets.
        if 0 != schedule_receives(&mut this.rx_ring, extras.ip.default_packet_pool()) {
            crate::write_reg!(enet, this.enet, RDAR, RDAR: 1);
        }

        this.tx_ring.schedule_transmits();
        crate::write_reg!(enet, this.enet, TDAR, TDAR: 1);

        Ok(())
    }

    fn send_packet(
        self: Pin<&'static mut Self>,
        extras: driver::DriverExtras<'static>,
        ethertype: u16,
    ) -> Result<(), driver::DriverError> {
        let Some(mut packet) = extras.packet else {
            return Err(driver::DriverError::Unhandled);
        };

        prepare_transmit_head(&mut packet, ethertype, extras.msw, extras.lsw);

        // Safety: we're not moving out of this object.
        let this = unsafe { self.get_unchecked_mut() };
        this.tx_ring.fifo_pending.push_back(packet);
        this.tx_ring.schedule_transmits();
        crate::write_reg!(enet, this.enet, TDAR, TDAR: 1);

        Ok(())
    }
}

const RECEIVE_INTERRUPT_EVENTS: u32 = enet::EIR::RXB::mask | enet::EIR::RXF::mask;
const TRANSMIT_INTERRUPT_EVENTS: u32 = enet::EIR::TXB::mask | enet::EIR::TXF::mask;

const DEFERRED_INTERRUPTS: NonZeroU32 =
    NonZeroU32::new(RECEIVE_INTERRUPT_EVENTS | TRANSMIT_INTERRUPT_EVENTS).unwrap();

/// Initialize the ENET IP block.
///
/// This assumes that you've already reset the IP block. When this
/// call returns, the ENET block is enabled. This is required for
/// MDIO interrupt activation. Although the IP block is enabled,
/// the call does not initialize any I/O.
fn initialize_enet(
    enet: Instance<enet::RegisterBlock>,
    tx_ring: &'static [TxBD],
    rx_ring: &'static [RxBD],
    rmii: bool,
) {
    crate::modify_reg!(enet, enet, ECR,
        DBSWP: 1,   // Swap data for this little endian device.
        EN1588: 1,  // Use enhanced buffer descriptors.
        RESET: 0,   // I think this auto-clears, but just in case...
        DBGEN: 0,   // Keep running the MAC in debug mode.
    );

    // Clear all interrupt flags.
    crate::write_reg!(enet, enet, EIR, u32::MAX);

    // Unmask interrupts.
    //
    // Make sure to update DEFERRED_INTERRUPTS if the event
    // needs to signal the IP thread.
    crate::write_reg!(enet, enet, EIMR,
        // MDIO completions.
        MII: 1,
        // Receive buffer complete.
        RXB: 1,
        // Receive frame complete.
        RXF: 1,
        // Transmit buffer complete.
        TXB: 1,
        // Transmit frame complete.
        TXF: 1,
    );

    // Tell the DMA engine which descriptors are the last
    // ones in the ring.
    if let Some(tx_tail) = tx_ring.last() {
        tx_tail
            .flags
            .fetch_or(enet::tx_bd::FLAGS_WRAP, Ordering::Relaxed);
    }
    if let Some(rx_tail) = rx_ring.last() {
        rx_tail
            .flags
            .fetch_or(enet::rx_bd::FLAGS_WRAP, Ordering::Relaxed);
    }

    // Establish the ring starting addresses.
    crate::write_reg!(enet, enet, TDSR, tx_ring.as_ptr() as u32);
    crate::write_reg!(enet, enet, RDSR, rx_ring.as_ptr() as u32);

    crate::modify_reg!(enet, enet, RCR,
        // Default max frame length without VLAN tags.
        MAX_FL: 1518,
        // Disable loopback by default. If we expose half-duplex to
        // the user, we wouldn't be able to support this.
        LOOP: 0,
        // No need to snoop.
        PROM: 0,
        // Do not reject broadcast frames; we might be interested
        // in these.
        BC_REJ: 0,
        // The MAC doesn't supply pause frames to the application.
        PAUFWD: 0,
        // Drop padding, along with the CRC, when supplying frames
        // to our software. This configuration implicitly includes
        // the CRC, so the CRCFWD below has no effect.
        PADEN: 1,
        // Drop the CRC in received frames. This doesn't turn off
        // CRC checking at the hardware level.
        //
        // If PADEN is set, this configuration does nothing.
        CRCFWD: 1,
        // Check the payload length based on the expected frame type /
        // frame length (encoded in the frame).
        NLC: 1,
        // Enable flow control; react to pause frames by pausing the data
        // transmit paths.
        FCE: 1,
        // MII or RMII mode; must be set.
        MII_MODE: 1,
        // Is this an RMII interface?
        RMII_MODE: rmii as u32,
        // Default to 100Mbit/sec.
        RMII_10T: 0,
        // For prototyping purposes, we're strictly a full-duplex MAC.
        DRT: 0,
    );

    crate::modify_reg!(enet, enet, TCR,
        // We told the IP thread to skip software CRCs. Let
        // the hardware handle it.
        CRCFWD: 0,
        // We'll program our source MAC into the PADDR registers.
        // It's our job to set up the frame, so we can choose to
        // have the hardware inject the source address.
        ADDINS: 1,
        // For prototyping purposes, we're strictly
        // a full-duplex MAC.
        FDEN: 1,
    );

    // Enable store-and-forward: start transmitting once you have a complete
    // frame in the FIFO.
    crate::modify_reg!(enet, enet, TFWR, STRFWD: 1);
    // Maintain store-and-forward on the receive path: use the receive queue
    // as a buffer until an entire frame is received.
    crate::write_reg!(enet, enet, RSFL, 0);

    // These accelerator options assume store-and-forward operations on both
    // data paths. See above.
    crate::modify_reg!(enet, enet, RACC,
        // Discard frames with MAC errors (checksumming, length, PHY errors).
        LINEDIS: 1,
        // Discard frames with the wrong checksums for the protocol and headers.
        PRODIS: 1,
        IPDIS: 1,
        // Discard any padding within a short IP datagram.
        PADREM: 1,
        // Insert two extra bytes so that the data section is four byte aligned.
        SHIFT16: 1,
    );
    crate::modify_reg!(enet, enet, TACC,
        // Enable protocol checksums. Assumes that the netx-duo sets these fields
        // to zero on our behalf.
        PROCHK: 1,
        // Enable IP checksum injection into the IPv4 header. Assumes that netx-duo
        // sets these fields to zero on our behalf.
        IPCHK: 1,
        // Expect two extra bytes when transmitting data.
        SHIFT16: 1,
    );

    // Enable the IP block.
    crate::modify_reg!(enet, enet, ECR, ETHEREN: 1);
}

/// Set up receive DMA operations for packet reception.
fn schedule_receives(rx_ring: &mut ReceiveRing, packet_pool: &'static PacketPool) -> usize {
    let mut data_capacity = 0;
    while rx_ring.is_schedulable()
        && let Some(packet) = packet_pool
            .try_allocate(packet::PacketType::Receive)
            .unwrap()
    {
        data_capacity = packet.data_capacity();
        assert_ne!(data_capacity, 0);
        assert!(data_capacity % 64 == 0);

        rx_ring.schedule_next(packet);
    }

    data_capacity
}

/// Give the received packet to the IP instance.
fn enqueue_rx_packet_to_ip(
    mut packet: packet::Packet<'static>,
    ip: &'static Ip,
    interface: Interface<'static>,
) {
    // Safety: Interface and packet have same lifetime.
    unsafe { packet.set_ip_interface(interface) };

    // Safety: Assuming no 802.1 tag, the Ethertype enum
    // starts at 6 + 6 bytes from the start of the data.
    // It uses network byte order.
    let ethertype = unsafe {
        let prepend_ptr = packet.prepend_ptr().add(12);
        (prepend_ptr.read() as u16) << 8 | prepend_ptr.add(1).read() as u16
    };

    if ![
        driver::ETHERTYPE_ARP,
        driver::ETHERTYPE_IPV4,
        driver::ETHERTYPE_IPV6,
        driver::ETHERTYPE_RARP,
    ]
    .contains(&ethertype)
    {
        return; // Packet drop deallocates the packet.
    }

    // We know how to handle this! Hide the Ethernet header from
    // the network stack.
    //
    // Safety: we know that we just recieved this data and that the
    // RX ring removes the two byte padding that starts this data.
    //
    // Safety: data remains in bounds. We're removing pointers and
    // info to access valid data.
    unsafe {
        packet.set_prepend_ptr(packet.prepend_ptr().add(driver::ETHERNET_FRAME_SIZE));
        packet.set_len(packet.len() - driver::ETHERNET_FRAME_SIZE);
    }

    if ethertype == driver::ETHERTYPE_IPV4 || ethertype == driver::ETHERTYPE_IPV6 {
        ip.defer_ip_receive(packet);
    } else if ethertype == driver::ETHERTYPE_ARP {
        ip.defer_arp_receive(packet);
    } else if ethertype == driver::ETHERTYPE_RARP {
        ip.defer_rarp_receive(packet);
    } else {
        unreachable!();
    }
}

fn prepare_transmit_head(packet: &mut packet::Packet<'_>, ethertype: u16, msw: u32, lsw: u32) {
    // Make space for an Ethernet header.
    //
    // Safety: user expected to have allocated this packet
    // appropriately.
    unsafe {
        packet.set_prepend_ptr(packet.prepend_ptr().sub(driver::ETHERNET_FRAME_SIZE));
        packet.set_len(packet.len() + driver::ETHERNET_FRAME_SIZE);
    }

    // Fill in the destination MAC and ethertype.
    //
    // Safety: pointer is in range for all accesses.
    unsafe {
        let prepend_ptr = packet.prepend_ptr();

        prepend_ptr.add(0).write((msw >> 8) as u8);
        prepend_ptr.add(1).write(msw as u8);
        prepend_ptr.add(2).write((lsw >> 24) as u8);
        prepend_ptr.add(3).write((lsw >> 16) as u8);
        prepend_ptr.add(4).write((lsw >> 8) as u8);
        prepend_ptr.add(5).write(lsw as u8);

        // MAC fills in the source address.

        prepend_ptr.add(12).write((ethertype >> 8) as u8);
        prepend_ptr.add(13).write(ethertype as u8);
    }

    // Make space for our two byte padding.
    //
    // Safety: NetX assumes a 16 byte allocation for the Ethernet
    // header. Therefore, this remains in range.
    unsafe {
        packet.set_prepend_ptr(packet.prepend_ptr().sub(2));
        packet.set_len(packet.len() + 2);
    }
}

/// Ring state for managing receive operations.
struct ReceiveRing {
    buffer_descriptors: &'static [RxBD],
    idx_in_flight: usize,
    idx_schedulable: usize,

    fifo_in_flight: PacketList<'static>,
    fifo_chaining: PacketChainer<'static>,
}

impl ReceiveRing {
    const fn new(buffer_descriptors: &'static [RxBD]) -> Self {
        Self {
            buffer_descriptors,
            idx_in_flight: 0,
            idx_schedulable: 0,
            fifo_in_flight: PacketList::empty(),
            fifo_chaining: PacketChainer::empty(),
        }
    }

    /// Do we have a descriptor for scheduling a receive?
    fn is_schedulable(&self) -> bool {
        let rx_bd = &self.buffer_descriptors[self.idx_schedulable];
        let flags = rx_bd.flags.load(Ordering::Relaxed);
        flags & enet::rx_bd::FLAGS_EMPTY == 0 && flags & enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1 == 0
    }

    /// Schedule a receive into the given packet.
    ///
    /// Assumes we have a schedulable descriptor. This updates
    /// the next schedulable descriptor.
    fn schedule_next(&mut self, packet: packet::Packet<'static>) {
        let rx_bd = &self.buffer_descriptors[self.idx_schedulable];

        rx_bd.data_length.store(0, Ordering::Relaxed);
        rx_bd
            .data_buffer_pointer
            .store(packet.prepend_ptr() as u32, Ordering::Relaxed);
        rx_bd
            .control
            .fetch_or(enet::rx_bd::CONTROL_INT, Ordering::Relaxed);
        rx_bd.flags.fetch_or(
            enet::rx_bd::FLAGS_EMPTY | enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1,
            Ordering::Relaxed,
        );

        atomic::fence(Ordering::Release);

        self.fifo_in_flight.push_back(packet);

        self.idx_schedulable = (self.idx_schedulable + 1) % self.buffer_descriptors.len();
    }

    /// Try to receive a frame with one or more packets.
    ///
    /// If this returns a packet, you should try calling it
    /// again; it may be able to produce another packet.
    fn try_frame_receive(&mut self) -> Option<packet::Packet<'static>> {
        // Show the borrow checker what we're doing.
        let Self {
            buffer_descriptors,
            idx_in_flight,
            fifo_in_flight,
            fifo_chaining,
            ..
        } = self;

        // Which receive operations have complete? Pop those packets so we can
        // start to form a packet chain.
        let completions = core::iter::from_fn(|| {
            let rx_bd = &buffer_descriptors[*idx_in_flight];
            let flags = rx_bd.flags.load(Ordering::Relaxed);
            let ready = !fifo_in_flight.is_empty()
                && flags & enet::rx_bd::FLAGS_EMPTY == 0
                && flags & enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1 != 0;

            ready.then(|| {
                rx_bd
                    .flags
                    .fetch_and(!enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1, Ordering::Relaxed);

                // Panic unlikely. We checked if the FIFO is empty
                // when deciding if it's ready.
                let packet = fifo_in_flight.pop_front().unwrap();
                *idx_in_flight = (*idx_in_flight + 1) % buffer_descriptors.len();

                let is_last = flags & enet::rx_bd::FLAGS_LAST != 0;
                let total_data_length: usize = rx_bd.data_length.load(Ordering::Relaxed).into();

                (total_data_length, packet, is_last)
            })
        })
        // Loop bound by the number of receive descriptors that could
        // possibly be filled. Without this, we could enter the loop,
        // observe that all descriptors are filled, and loop forever.
        // We must eventually break to prime another batch of descriptors.
        .take(buffer_descriptors.len());

        for (total_data_length, mut packet, is_last) in completions {
            // Per the ENET docs, when the last flag is set, the data length describes
            // the total length of the entire frame. We only use this when figuring out
            // information for the last packet. Note that this includes the two bytes
            // of padding.

            if is_last {
                // The number of other packets in the chain,
                // besides this one. This packet hasn't been
                // inserted into the chain, yet, so this can
                // be zero, signaling "only packet."
                let other_packets = fifo_chaining.len();

                // If there are other packets, they're saturated
                // to capacity.
                let data_in_other_packets = other_packets * packet.data_capacity();

                // The data in the final (only) packet is the total length
                // without the data in other packets. This includes the
                // two bytes of padding.
                let our_data_len = total_data_length - data_in_other_packets;

                // Safety: computation of data in the final packet maintains
                // an in-bounds offset into the packet's data. It's known that
                // a receive packet is allocated with its data start and prepend
                // pointers pointing at the same place.
                //
                // If this is an only packet, then the offset already accounts
                // for the two byte padding that we'll strip from the prepend
                // pointer, later. If this is the tail of the packet chain, then
                // we already need to handle the extra two bytes that aren't in
                // the chain's head packet.
                unsafe {
                    packet.set_append_ptr(packet.prepend_ptr().add(our_data_len));
                }
            } else {
                // Safety: This is an intermediate packet in the chain.
                // Since it's not a last packet, it's not an only packet.
                // Therefore, it's been filled to capacity by the DMA
                // engine.
                //
                // Since it's been filled to capacity, the data end pointer
                // represents valid data. Module inspection shows that the
                // maximum buffer size is the packet capacity. It's known
                // that a receive type pointer has its data start and prepend
                // pointers pointing at the same address.
                unsafe {
                    packet.set_append_ptr(packet.data_end());
                }
            }

            // Insert all packets into the current chain.
            // Once we see the last packet, we'll pop the
            // chains head, clean it up, and hand it off.
            fifo_chaining.push_back(packet);
            if is_last {
                // Panic unlikely. We just pushed a packet into the FIFO, before
                // the branch.
                let mut packet = fifo_chaining.pop().unwrap();

                // Safety: We're trusting the hardware to represent the total
                // frame size in this descriptor. We remove the two bytes of
                // padding inserted as data.
                unsafe { packet.set_len(total_data_length - 2) };

                // Safety: prepend pointer and its two byte offset are part of
                // the same head packet. We're simply telling the netstack
                // to skip the invalid two bytes in the front of the packet.
                unsafe { packet.set_prepend_ptr(packet.prepend_ptr().add(2)) };

                // We formed a chained packet. Return it to the user.
                //
                // If there's more data to process, the user can keep
                // calling us.
                return Some(packet);
            }
        }

        // Nothing was ready.
        None
    }
}

/// Ring state for managing transmits.
struct TransmitRing {
    buffer_descriptors: &'static [TxBD],

    idx_schedulable: usize,
    idx_in_flight: usize,

    /// Packets that need to be assigned
    /// to a descriptor.
    fifo_pending: PacketListChainView<'static>,
    /// Packets that are in flight.
    fifo_in_flight: PacketList<'static>,
}

impl TransmitRing {
    const fn new(buffer_descriptors: &'static [TxBD]) -> Self {
        Self {
            buffer_descriptors,
            idx_schedulable: 0,
            idx_in_flight: 0,
            fifo_pending: PacketListChainView::empty(),
            fifo_in_flight: PacketList::empty(),
        }
    }

    /// Try scheduling transmits for a chain of packets.
    ///
    /// Returns the packet once all packets in its chain
    /// have been scheduled. Otherwise, returns `None` if
    /// there are packets in the chain that still need
    /// scheduling. Keep calling this in a loop to drive
    /// packet scheduling.
    fn try_schedule_packet_chain(&mut self) -> Option<packet::Packet<'static>> {
        // Flags set for the last packet in the frame.
        const LAST_FRAME_FLAGS: u16 =
            enet::tx_bd::FLAGS_LAST_IN_FRAME | enet::tx_bd::FLAGS_TRANSMIT_CRC;

        let Self {
            buffer_descriptors,
            idx_schedulable,
            fifo_pending,
            ..
        } = self;

        // True while we're iterating over
        // a chain of packets.
        while fifo_pending.has_chain_link() {
            let tx_bd = &buffer_descriptors[*idx_schedulable];

            if tx_bd.flags.load(Ordering::Relaxed) & enet::tx_bd::FLAGS_READY != 0 {
                // Descriptor is waiting to transmit.
                return None;
            }

            // Schedule a DMA transfer from the chained packet we're
            // looking at.
            fifo_pending.with_chain_link(|packet| {
                if packet.has_chained_packet() {
                    // Clear any "last packet" flags set by the prior transfer.
                    tx_bd.flags.fetch_and(!LAST_FRAME_FLAGS, Ordering::Relaxed);
                } else {
                    // This is the final packet in the chain.
                    tx_bd.flags.fetch_or(LAST_FRAME_FLAGS, Ordering::Relaxed);
                }

                tx_bd
                    .data_buffer_pointer
                    .store(packet.prepend_ptr() as u32, Ordering::Relaxed);
                tx_bd
                    .data_length
                    .store(packet.data_length() as u16, Ordering::Relaxed);
                tx_bd
                    .control
                    .fetch_or(enet::tx_bd::CONTROL_INT, Ordering::Relaxed);
                tx_bd
                    .flags
                    .fetch_or(enet::tx_bd::FLAGS_READY, Ordering::Relaxed);

                atomic::fence(Ordering::Release);
            });

            // We used a buffer descriptor. Go to the next one.
            *idx_schedulable = (*idx_schedulable + 1) % buffer_descriptors.len();

            // If this was the last packet in the chain,
            // we'll break the loop. Otherwise, we'll keep
            // looping while we have ready descriptors.
            fifo_pending.advance_chain_link();
        }

        // Either pops the packet chain that's been
        // scheduled. Or, it pops None when there's
        // nothing in the FIFO.
        return fifo_pending.pop_front();
    }

    /// Try to schedule transmit operations on pending packets.
    fn schedule_transmits(&mut self) {
        while let Some(packet) = self.try_schedule_packet_chain() {
            self.fifo_in_flight.push_back(packet);
        }
    }

    /// Release descriptors and packets that have completed transmission.
    fn deallocate_completions(&mut self) {
        let completions = core::iter::from_fn(|| {
            let tx_bd = &self.buffer_descriptors[self.idx_in_flight];
            let flags = tx_bd.flags.load(Ordering::Relaxed);

            let complete = !self.fifo_in_flight.is_empty() && flags & enet::tx_bd::FLAGS_READY == 0;

            complete.then(|| {
                self.idx_in_flight = (self.idx_in_flight + 1) % self.buffer_descriptors.len();

                let last = flags & enet::tx_bd::FLAGS_LAST_IN_FRAME != 0;
                last.then(|| self.fifo_in_flight.pop_front().unwrap())
            })
        })
        // Bound the number of descriptors that could possibly
        // complete in one evaluation. Without this, there's a
        // chance to loop endlessly when the number of packets
        // exceeds the number of available descriptors. Break
        // the loop to continue packet scheduling.
        .take(self.buffer_descriptors.len());

        // Loop runs for every complete packet, which may
        // not be the last packet. Drive the completions as
        // far as possible so we can deallocate the packet
        // chain ASAP.
        for packet in completions {
            if let Some(mut packet) = packet {
                // Strip the Ethernet frame and padding before deallocation.
                //
                // Safety: pointer remains in bounds of an allocation.
                unsafe {
                    packet
                        .set_prepend_ptr(packet.prepend_ptr().add(driver::ETHERNET_FRAME_SIZE + 2));
                    packet.set_len(packet.len() - driver::ETHERNET_FRAME_SIZE - 2);
                }

                // Try to release a packet that may need retransmission.
                //
                // If the release didn't happen, we'll be given back the
                // packet through another send call.
                packet.transmit_release();
            }
        }
    }
}