diff options
Diffstat (limited to 'crates/imxrt/src/drivers/enet.rs')
| -rw-r--r-- | crates/imxrt/src/drivers/enet.rs | 849 |
1 files changed, 849 insertions, 0 deletions
diff --git a/crates/imxrt/src/drivers/enet.rs b/crates/imxrt/src/drivers/enet.rs new file mode 100644 index 0000000..611fa51 --- /dev/null +++ b/crates/imxrt/src/drivers/enet.rs @@ -0,0 +1,849 @@ +use crate::Instance; +use crate::ral::enet; +use core::{ + num::NonZeroU32, + pin::Pin, + sync::atomic::{self, Ordering}, +}; + +use fusible::{ + event_flags::{EventFlags, EventFlagsContext, GetOption, SetOption}, + netx_duo::{ + driver, + interface::{Capability, Interface}, + ip::Ip, + packet::{self, PacketChainer, PacketList, PacketListChainView, PacketPool}, + }, +}; + +use rust_threadx_net_phys::{Mdio, Phy}; + +use super::gpio::GpioOutput; + +pub use enet::{RxRing, TxRing, rx_bd::RxBD, tx_bd::TxBD}; + +pub struct EnetContext { + flags: EventFlagsContext<'static>, +} + +const MII_EVENT: NonZeroU32 = NonZeroU32::new(enet::EIR::MII::mask).unwrap(); + +impl EnetContext { + pub const fn new() -> Self { + Self { + flags: EventFlags::context(), + } + } + + #[inline(always)] + pub unsafe fn on_interrupt( + &'static self, + enet: Instance<enet::RegisterBlock>, + ip: &'static Ip, + ) { + unsafe { + let eir = crate::read_reg!(enet, enet, EIR); + crate::write_reg!(enet, enet, EIR, eir); + + if eir & DEFERRED_INTERRUPTS.get() != 0 { + ip.deferred_processing(); + } + + Pin::static_ref(&self.flags) + .assume_created() + .set(eir, SetOption::Or); + } + } + + pub unsafe fn create<P: Phy<EnetMdio>>( + &'static self, + enet: Instance<enet::RegisterBlock>, + tx_ring: &'static [TxBD], + rx_ring: &'static [RxBD], + phy: P, + rst: GpioOutput, + mdio_source_clock_hz: u32, + ) -> Enet<P> { + let flags = EventFlags::create(Pin::static_ref(&self.flags), &Default::default()).unwrap(); + + crate::write_reg!(enet, enet, ECR, RESET: 1); + + const SMI_MDC_FREQUENCY_HZ: u32 = 2_500_000; + let mii_speed = mdio_source_clock_hz.div_ceil(2 * SMI_MDC_FREQUENCY_HZ) - 1; + let hold_time = 10_u32.div_ceil(1_000_000_000 / mdio_source_clock_hz) - 1; + crate::modify_reg!(enet, enet, MSCR, HOLDTIME: hold_time, MII_SPEED: mii_speed); + + Enet { + enet, + phy, + rst, + flags, + + tx_ring: TransmitRing::new(tx_ring), + rx_ring: ReceiveRing::new(rx_ring), + } + } +} + +pub struct EnetMdio { + enet: Instance<enet::RegisterBlock>, + flags: &'static EventFlags, +} + +pub struct Enet<P: Phy<EnetMdio>> { + enet: Instance<enet::RegisterBlock>, + phy: P, + rst: GpioOutput, + flags: &'static EventFlags, + + tx_ring: TransmitRing, + rx_ring: ReceiveRing, +} + +unsafe impl<P: Phy<EnetMdio>> Send for Enet<P> {} + +impl Mdio for EnetMdio { + type Error = core::convert::Infallible; + + fn read(&mut self, ctrl: u16) -> Result<u16, Self::Error> { + let mmfr = (ctrl as u32) << 16; + crate::write_reg!(enet, self.enet, MMFR, mmfr); + + self.flags.get(MII_EVENT, GetOption::AndClear).unwrap(); + + // Automatically discards control bits. + let data = crate::read_reg!(enet, self.enet, MMFR, DATA) as u16; + Ok(data) + } + + fn write(&mut self, ctrl: u16, data: u16) -> Result<(), Self::Error> { + let mmfr = (ctrl as u32) << 16 | data as u32; + crate::write_reg!(enet, self.enet, MMFR, mmfr); + + self.flags.get(MII_EVENT, GetOption::AndClear).unwrap(); + + Ok(()) + } +} + +impl<P: Phy<EnetMdio>> driver::Driver<'static> for Enet<P> { + fn attach_interface( + self: Pin<&'static mut Self>, + mut extras: driver::DriverExtras<'static>, + ) -> Result<(), driver::DriverError> { + extras + .interface + .set_capability( + extras.ip, + Capability::IPV4_RX_CHECKSUM + | Capability::IPV4_TX_CHECKSUM + | Capability::UDP_RX_CHECKSUM + | Capability::UDP_TX_CHECKSUM + | Capability::TCP_RX_CHECKSUM + | Capability::TCP_TX_CHECKSUM, + ) + .unwrap(); + + // TODO: this may need to be bound by MRBR. Otherwise, + // we're lying to the upper levels. + extras.interface.set_mtu(extras.ip, 1500); + extras.interface.driver_needs_mapping(extras.ip); + + Ok(()) + } + + fn initialize_link( + self: Pin<&'static mut Self>, + _: driver::DriverExtras<'static>, + ) -> Result<(), driver::DriverError> { + // Safety: not relocating anything in memory. + let this = unsafe { self.get_unchecked_mut() }; + initialize_enet( + this.enet, + this.tx_ring.buffer_descriptors, + this.rx_ring.buffer_descriptors, + true, + ); + + this.rst.clear(); + fusible::thread::sleep(10); + this.rst.set(); + fusible::thread::sleep(10); + + let mut mdio = EnetMdio { + enet: this.enet.clone(), + flags: this.flags, + }; + + this.phy.initialize(&mut mdio).unwrap(); + + Ok(()) + } + + fn enable_link( + self: Pin<&'static mut Self>, + mut extras: driver::DriverExtras<'static>, + ) -> Result<(), driver::DriverError> { + // Safety: not moving out of self. + let this = unsafe { self.get_unchecked_mut() }; + let mrbr = schedule_receives(&mut this.rx_ring, extras.ip.default_packet_pool()); + assert_ne!(mrbr, 0); + + crate::write_reg!(enet, this.enet, MRBR, mrbr as u32); + crate::write_reg!(enet, this.enet, RDAR, RDAR: 1); + + extras.interface.set_link_up(true); + + Ok(()) + } + + fn set_physical_address( + self: Pin<&'static mut Self>, + extras: driver::DriverExtras<'static>, + ) -> Result<(), driver::DriverError> { + crate::write_reg!(enet, self.enet, PALR, extras.msw << 16 | (extras.lsw >> 16)); + crate::write_reg!(enet, self.enet, PAUR, extras.lsw << 16); + Ok(()) + } + + fn deferred_processing( + self: Pin<&'static mut Self>, + extras: driver::DriverExtras<'static>, + ) -> Result<(), driver::DriverError> { + // Safety: not moving out of this object. + let this = unsafe { self.get_unchecked_mut() }; + let eir = this.flags.try_get(DEFERRED_INTERRUPTS, GetOption::OrClear); + let eir = eir.map_or(0, NonZeroU32::get); + + if eir & TRANSMIT_INTERRUPT_EVENTS != 0 { + this.tx_ring.deallocate_completions(); + } + + if eir & RECEIVE_INTERRUPT_EVENTS != 0 { + while let Some(packet) = this.rx_ring.try_frame_receive() { + enqueue_rx_packet_to_ip(packet, extras.ip, extras.interface); + } + } + + // Make sure receive operations are primed. Otherwise, + // we drop data. Once all descriptors are saturated, + // try transmitting pending packets. + if 0 != schedule_receives(&mut this.rx_ring, extras.ip.default_packet_pool()) { + crate::write_reg!(enet, this.enet, RDAR, RDAR: 1); + } + + this.tx_ring.schedule_transmits(); + crate::write_reg!(enet, this.enet, TDAR, TDAR: 1); + + Ok(()) + } + + fn send_packet( + self: Pin<&'static mut Self>, + extras: driver::DriverExtras<'static>, + ethertype: u16, + ) -> Result<(), driver::DriverError> { + let Some(mut packet) = extras.packet else { + return Err(driver::DriverError::Unhandled); + }; + + prepare_transmit_head(&mut packet, ethertype, extras.msw, extras.lsw); + + // Safety: we're not moving out of this object. + let this = unsafe { self.get_unchecked_mut() }; + this.tx_ring.fifo_pending.push_back(packet); + this.tx_ring.schedule_transmits(); + crate::write_reg!(enet, this.enet, TDAR, TDAR: 1); + + Ok(()) + } +} + +const RECEIVE_INTERRUPT_EVENTS: u32 = enet::EIR::RXB::mask | enet::EIR::RXF::mask; +const TRANSMIT_INTERRUPT_EVENTS: u32 = enet::EIR::TXB::mask | enet::EIR::TXF::mask; + +const DEFERRED_INTERRUPTS: NonZeroU32 = + NonZeroU32::new(RECEIVE_INTERRUPT_EVENTS | TRANSMIT_INTERRUPT_EVENTS).unwrap(); + +/// Initialize the ENET IP block. +/// +/// This assumes that you've already reset the IP block. When this +/// call returns, the ENET block is enabled. This is required for +/// MDIO interrupt activation. Although the IP block is enabled, +/// the call does not initialize any I/O. +fn initialize_enet( + enet: Instance<enet::RegisterBlock>, + tx_ring: &'static [TxBD], + rx_ring: &'static [RxBD], + rmii: bool, +) { + crate::modify_reg!(enet, enet, ECR, + DBSWP: 1, // Swap data for this little endian device. + EN1588: 1, // Use enhanced buffer descriptors. + RESET: 0, // I think this auto-clears, but just in case... + DBGEN: 0, // Keep running the MAC in debug mode. + ); + + // Clear all interrupt flags. + crate::write_reg!(enet, enet, EIR, u32::MAX); + + // Unmask interrupts. + // + // Make sure to update DEFERRED_INTERRUPTS if the event + // needs to signal the IP thread. + crate::write_reg!(enet, enet, EIMR, + // MDIO completions. + MII: 1, + // Receive buffer complete. + RXB: 1, + // Receive frame complete. + RXF: 1, + // Transmit buffer complete. + TXB: 1, + // Transmit frame complete. + TXF: 1, + ); + + // Tell the DMA engine which descriptors are the last + // ones in the ring. + if let Some(tx_tail) = tx_ring.last() { + tx_tail + .flags + .fetch_or(enet::tx_bd::FLAGS_WRAP, Ordering::Relaxed); + } + if let Some(rx_tail) = rx_ring.last() { + rx_tail + .flags + .fetch_or(enet::rx_bd::FLAGS_WRAP, Ordering::Relaxed); + } + + // Establish the ring starting addresses. + crate::write_reg!(enet, enet, TDSR, tx_ring.as_ptr() as u32); + crate::write_reg!(enet, enet, RDSR, rx_ring.as_ptr() as u32); + + crate::modify_reg!(enet, enet, RCR, + // Default max frame length without VLAN tags. + MAX_FL: 1518, + // Disable loopback by default. If we expose half-duplex to + // the user, we wouldn't be able to support this. + LOOP: 0, + // No need to snoop. + PROM: 0, + // Do not reject broadcast frames; we might be interested + // in these. + BC_REJ: 0, + // The MAC doesn't supply pause frames to the application. + PAUFWD: 0, + // Drop padding, along with the CRC, when supplying frames + // to our software. This configuration implicitly includes + // the CRC, so the CRCFWD below has no effect. + PADEN: 1, + // Drop the CRC in received frames. This doesn't turn off + // CRC checking at the hardware level. + // + // If PADEN is set, this configuration does nothing. + CRCFWD: 1, + // Check the payload length based on the expected frame type / + // frame length (encoded in the frame). + NLC: 1, + // Enable flow control; react to pause frames by pausing the data + // transmit paths. + FCE: 1, + // MII or RMII mode; must be set. + MII_MODE: 1, + // Is this an RMII interface? + RMII_MODE: rmii as u32, + // Default to 100Mbit/sec. + RMII_10T: 0, + // For prototyping purposes, we're strictly a full-duplex MAC. + DRT: 0, + ); + + crate::modify_reg!(enet, enet, TCR, + // We told the IP thread to skip software CRCs. Let + // the hardware handle it. + CRCFWD: 0, + // We'll program our source MAC into the PADDR registers. + // It's our job to set up the frame, so we can choose to + // have the hardware inject the source address. + ADDINS: 1, + // For prototyping purposes, we're strictly + // a full-duplex MAC. + FDEN: 1, + ); + + // Enable store-and-forward: start transmitting once you have a complete + // frame in the FIFO. + crate::modify_reg!(enet, enet, TFWR, STRFWD: 1); + // Maintain store-and-forward on the receive path: use the receive queue + // as a buffer until an entire frame is received. + crate::write_reg!(enet, enet, RSFL, 0); + + // These accelerator options assume store-and-forward operations on both + // data paths. See above. + crate::modify_reg!(enet, enet, RACC, + // Discard frames with MAC errors (checksumming, length, PHY errors). + LINEDIS: 1, + // Discard frames with the wrong checksums for the protocol and headers. + PRODIS: 1, + IPDIS: 1, + // Discard any padding within a short IP datagram. + PADREM: 1, + // Insert two extra bytes so that the data section is four byte aligned. + SHIFT16: 1, + ); + crate::modify_reg!(enet, enet, TACC, + // Enable protocol checksums. Assumes that the netx-duo sets these fields + // to zero on our behalf. + PROCHK: 1, + // Enable IP checksum injection into the IPv4 header. Assumes that netx-duo + // sets these fields to zero on our behalf. + IPCHK: 1, + // Expect two extra bytes when transmitting data. + SHIFT16: 1, + ); + + // Enable the IP block. + crate::modify_reg!(enet, enet, ECR, ETHEREN: 1); +} + +/// Set up receive DMA operations for packet reception. +fn schedule_receives(rx_ring: &mut ReceiveRing, packet_pool: &'static PacketPool) -> usize { + let mut data_capacity = 0; + while rx_ring.is_schedulable() + && let Some(packet) = packet_pool + .try_allocate(packet::PacketType::Receive) + .unwrap() + { + data_capacity = packet.data_capacity(); + assert_ne!(data_capacity, 0); + assert!(data_capacity % 64 == 0); + + rx_ring.schedule_next(packet); + } + + data_capacity +} + +/// Give the received packet to the IP instance. +fn enqueue_rx_packet_to_ip( + mut packet: packet::Packet<'static>, + ip: &'static Ip, + interface: Interface<'static>, +) { + // Safety: Interface and packet have same lifetime. + unsafe { packet.set_ip_interface(interface) }; + + // Safety: Assuming no 802.1 tag, the Ethertype enum + // starts at 6 + 6 bytes from the start of the data. + // It uses network byte order. + let ethertype = unsafe { + let prepend_ptr = packet.prepend_ptr().add(12); + (prepend_ptr.read() as u16) << 8 | prepend_ptr.add(1).read() as u16 + }; + + if ![ + driver::ETHERTYPE_ARP, + driver::ETHERTYPE_IPV4, + driver::ETHERTYPE_IPV6, + driver::ETHERTYPE_RARP, + ] + .contains(ðertype) + { + return; // Packet drop deallocates the packet. + } + + // We know how to handle this! Hide the Ethernet header from + // the network stack. + // + // Safety: we know that we just recieved this data and that the + // RX ring removes the two byte padding that starts this data. + // + // Safety: data remains in bounds. We're removing pointers and + // info to access valid data. + unsafe { + packet.set_prepend_ptr(packet.prepend_ptr().add(driver::ETHERNET_FRAME_SIZE)); + packet.set_len(packet.len() - driver::ETHERNET_FRAME_SIZE); + } + + if ethertype == driver::ETHERTYPE_IPV4 || ethertype == driver::ETHERTYPE_IPV6 { + ip.defer_ip_receive(packet); + } else if ethertype == driver::ETHERTYPE_ARP { + ip.defer_arp_receive(packet); + } else if ethertype == driver::ETHERTYPE_RARP { + ip.defer_rarp_receive(packet); + } else { + unreachable!(); + } +} + +fn prepare_transmit_head(packet: &mut packet::Packet<'_>, ethertype: u16, msw: u32, lsw: u32) { + // Make space for an Ethernet header. + // + // Safety: user expected to have allocated this packet + // appropriately. + unsafe { + packet.set_prepend_ptr(packet.prepend_ptr().sub(driver::ETHERNET_FRAME_SIZE)); + packet.set_len(packet.len() + driver::ETHERNET_FRAME_SIZE); + } + + // Fill in the destination MAC and ethertype. + // + // Safety: pointer is in range for all accesses. + unsafe { + let prepend_ptr = packet.prepend_ptr(); + + prepend_ptr.add(0).write((msw >> 8) as u8); + prepend_ptr.add(1).write(msw as u8); + prepend_ptr.add(2).write((lsw >> 24) as u8); + prepend_ptr.add(3).write((lsw >> 16) as u8); + prepend_ptr.add(4).write((lsw >> 8) as u8); + prepend_ptr.add(5).write(lsw as u8); + + // MAC fills in the source address. + + prepend_ptr.add(12).write((ethertype >> 8) as u8); + prepend_ptr.add(13).write(ethertype as u8); + } + + // Make space for our two byte padding. + // + // Safety: NetX assumes a 16 byte allocation for the Ethernet + // header. Therefore, this remains in range. + unsafe { + packet.set_prepend_ptr(packet.prepend_ptr().sub(2)); + packet.set_len(packet.len() + 2); + } +} + +/// Ring state for managing receive operations. +struct ReceiveRing { + buffer_descriptors: &'static [RxBD], + idx_in_flight: usize, + idx_schedulable: usize, + + fifo_in_flight: PacketList<'static>, + fifo_chaining: PacketChainer<'static>, +} + +impl ReceiveRing { + const fn new(buffer_descriptors: &'static [RxBD]) -> Self { + Self { + buffer_descriptors, + idx_in_flight: 0, + idx_schedulable: 0, + fifo_in_flight: PacketList::empty(), + fifo_chaining: PacketChainer::empty(), + } + } + + /// Do we have a descriptor for scheduling a receive? + fn is_schedulable(&self) -> bool { + let rx_bd = &self.buffer_descriptors[self.idx_schedulable]; + let flags = rx_bd.flags.load(Ordering::Relaxed); + flags & enet::rx_bd::FLAGS_EMPTY == 0 && flags & enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1 == 0 + } + + /// Schedule a receive into the given packet. + /// + /// Assumes we have a schedulable descriptor. This updates + /// the next schedulable descriptor. + fn schedule_next(&mut self, packet: packet::Packet<'static>) { + let rx_bd = &self.buffer_descriptors[self.idx_schedulable]; + + rx_bd.data_length.store(0, Ordering::Relaxed); + rx_bd + .data_buffer_pointer + .store(packet.prepend_ptr() as u32, Ordering::Relaxed); + rx_bd + .control + .fetch_or(enet::rx_bd::CONTROL_INT, Ordering::Relaxed); + rx_bd.flags.fetch_or( + enet::rx_bd::FLAGS_EMPTY | enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1, + Ordering::Relaxed, + ); + + atomic::fence(Ordering::Release); + + self.fifo_in_flight.push_back(packet); + + self.idx_schedulable = (self.idx_schedulable + 1) % self.buffer_descriptors.len(); + } + + /// Try to receive a frame with one or more packets. + /// + /// If this returns a packet, you should try calling it + /// again; it may be able to produce another packet. + fn try_frame_receive(&mut self) -> Option<packet::Packet<'static>> { + // Show the borrow checker what we're doing. + let Self { + buffer_descriptors, + idx_in_flight, + fifo_in_flight, + fifo_chaining, + .. + } = self; + + // Which receive operations have complete? Pop those packets so we can + // start to form a packet chain. + let completions = core::iter::from_fn(|| { + let rx_bd = &buffer_descriptors[*idx_in_flight]; + let flags = rx_bd.flags.load(Ordering::Relaxed); + let ready = !fifo_in_flight.is_empty() + && flags & enet::rx_bd::FLAGS_EMPTY == 0 + && flags & enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1 != 0; + + ready.then(|| { + rx_bd + .flags + .fetch_and(!enet::rx_bd::FLAGS_RECEIVE_OWNERSHP_1, Ordering::Relaxed); + + // Panic unlikely. We checked if the FIFO is empty + // when deciding if it's ready. + let packet = fifo_in_flight.pop_front().unwrap(); + *idx_in_flight = (*idx_in_flight + 1) % buffer_descriptors.len(); + + let is_last = flags & enet::rx_bd::FLAGS_LAST != 0; + let total_data_length: usize = rx_bd.data_length.load(Ordering::Relaxed).into(); + + (total_data_length, packet, is_last) + }) + }) + // Loop bound by the number of receive descriptors that could + // possibly be filled. Without this, we could enter the loop, + // observe that all descriptors are filled, and loop forever. + // We must eventually break to prime another batch of descriptors. + .take(buffer_descriptors.len()); + + for (total_data_length, mut packet, is_last) in completions { + // Per the ENET docs, when the last flag is set, the data length describes + // the total length of the entire frame. We only use this when figuring out + // information for the last packet. Note that this includes the two bytes + // of padding. + + if is_last { + // The number of other packets in the chain, + // besides this one. This packet hasn't been + // inserted into the chain, yet, so this can + // be zero, signaling "only packet." + let other_packets = fifo_chaining.len(); + + // If there are other packets, they're saturated + // to capacity. + let data_in_other_packets = other_packets * packet.data_capacity(); + + // The data in the final (only) packet is the total length + // without the data in other packets. This includes the + // two bytes of padding. + let our_data_len = total_data_length - data_in_other_packets; + + // Safety: computation of data in the final packet maintains + // an in-bounds offset into the packet's data. It's known that + // a receive packet is allocated with its data start and prepend + // pointers pointing at the same place. + // + // If this is an only packet, then the offset already accounts + // for the two byte padding that we'll strip from the prepend + // pointer, later. If this is the tail of the packet chain, then + // we already need to handle the extra two bytes that aren't in + // the chain's head packet. + unsafe { + packet.set_append_ptr(packet.prepend_ptr().add(our_data_len)); + } + } else { + // Safety: This is an intermediate packet in the chain. + // Since it's not a last packet, it's not an only packet. + // Therefore, it's been filled to capacity by the DMA + // engine. + // + // Since it's been filled to capacity, the data end pointer + // represents valid data. Module inspection shows that the + // maximum buffer size is the packet capacity. It's known + // that a receive type pointer has its data start and prepend + // pointers pointing at the same address. + unsafe { + packet.set_append_ptr(packet.data_end()); + } + } + + // Insert all packets into the current chain. + // Once we see the last packet, we'll pop the + // chains head, clean it up, and hand it off. + fifo_chaining.push_back(packet); + if is_last { + // Panic unlikely. We just pushed a packet into the FIFO, before + // the branch. + let mut packet = fifo_chaining.pop().unwrap(); + + // Safety: We're trusting the hardware to represent the total + // frame size in this descriptor. We remove the two bytes of + // padding inserted as data. + unsafe { packet.set_len(total_data_length - 2) }; + + // Safety: prepend pointer and its two byte offset are part of + // the same head packet. We're simply telling the netstack + // to skip the invalid two bytes in the front of the packet. + unsafe { packet.set_prepend_ptr(packet.prepend_ptr().add(2)) }; + + // We formed a chained packet. Return it to the user. + // + // If there's more data to process, the user can keep + // calling us. + return Some(packet); + } + } + + // Nothing was ready. + None + } +} + +/// Ring state for managing transmits. +struct TransmitRing { + buffer_descriptors: &'static [TxBD], + + idx_schedulable: usize, + idx_in_flight: usize, + + /// Packets that need to be assigned + /// to a descriptor. + fifo_pending: PacketListChainView<'static>, + /// Packets that are in flight. + fifo_in_flight: PacketList<'static>, +} + +impl TransmitRing { + const fn new(buffer_descriptors: &'static [TxBD]) -> Self { + Self { + buffer_descriptors, + idx_schedulable: 0, + idx_in_flight: 0, + fifo_pending: PacketListChainView::empty(), + fifo_in_flight: PacketList::empty(), + } + } + + /// Try scheduling transmits for a chain of packets. + /// + /// Returns the packet once all packets in its chain + /// have been scheduled. Otherwise, returns `None` if + /// there are packets in the chain that still need + /// scheduling. Keep calling this in a loop to drive + /// packet scheduling. + fn try_schedule_packet_chain(&mut self) -> Option<packet::Packet<'static>> { + // Flags set for the last packet in the frame. + const LAST_FRAME_FLAGS: u16 = + enet::tx_bd::FLAGS_LAST_IN_FRAME | enet::tx_bd::FLAGS_TRANSMIT_CRC; + + let Self { + buffer_descriptors, + idx_schedulable, + fifo_pending, + .. + } = self; + + // True while we're iterating over + // a chain of packets. + while fifo_pending.has_chain_link() { + let tx_bd = &buffer_descriptors[*idx_schedulable]; + + if tx_bd.flags.load(Ordering::Relaxed) & enet::tx_bd::FLAGS_READY != 0 { + // Descriptor is waiting to transmit. + return None; + } + + // Schedule a DMA transfer from the chained packet we're + // looking at. + fifo_pending.with_chain_link(|packet| { + if packet.has_chained_packet() { + // Clear any "last packet" flags set by the prior transfer. + tx_bd.flags.fetch_and(!LAST_FRAME_FLAGS, Ordering::Relaxed); + } else { + // This is the final packet in the chain. + tx_bd.flags.fetch_or(LAST_FRAME_FLAGS, Ordering::Relaxed); + } + + tx_bd + .data_buffer_pointer + .store(packet.prepend_ptr() as u32, Ordering::Relaxed); + tx_bd + .data_length + .store(packet.data_length() as u16, Ordering::Relaxed); + tx_bd + .control + .fetch_or(enet::tx_bd::CONTROL_INT, Ordering::Relaxed); + tx_bd + .flags + .fetch_or(enet::tx_bd::FLAGS_READY, Ordering::Relaxed); + + atomic::fence(Ordering::Release); + }); + + // We used a buffer descriptor. Go to the next one. + *idx_schedulable = (*idx_schedulable + 1) % buffer_descriptors.len(); + + // If this was the last packet in the chain, + // we'll break the loop. Otherwise, we'll keep + // looping while we have ready descriptors. + fifo_pending.advance_chain_link(); + } + + // Either pops the packet chain that's been + // scheduled. Or, it pops None when there's + // nothing in the FIFO. + return fifo_pending.pop_front(); + } + + /// Try to schedule transmit operations on pending packets. + fn schedule_transmits(&mut self) { + while let Some(packet) = self.try_schedule_packet_chain() { + self.fifo_in_flight.push_back(packet); + } + } + + /// Release descriptors and packets that have completed transmission. + fn deallocate_completions(&mut self) { + let completions = core::iter::from_fn(|| { + let tx_bd = &self.buffer_descriptors[self.idx_in_flight]; + let flags = tx_bd.flags.load(Ordering::Relaxed); + + let complete = !self.fifo_in_flight.is_empty() && flags & enet::tx_bd::FLAGS_READY == 0; + + complete.then(|| { + self.idx_in_flight = (self.idx_in_flight + 1) % self.buffer_descriptors.len(); + + let last = flags & enet::tx_bd::FLAGS_LAST_IN_FRAME != 0; + last.then(|| self.fifo_in_flight.pop_front().unwrap()) + }) + }) + // Bound the number of descriptors that could possibly + // complete in one evaluation. Without this, there's a + // chance to loop endlessly when the number of packets + // exceeds the number of available descriptors. Break + // the loop to continue packet scheduling. + .take(self.buffer_descriptors.len()); + + // Loop runs for every complete packet, which may + // not be the last packet. Drive the completions as + // far as possible so we can deallocate the packet + // chain ASAP. + for packet in completions { + if let Some(mut packet) = packet { + // Strip the Ethernet frame and padding before deallocation. + // + // Safety: pointer remains in bounds of an allocation. + unsafe { + packet + .set_prepend_ptr(packet.prepend_ptr().add(driver::ETHERNET_FRAME_SIZE + 2)); + packet.set_len(packet.len() - driver::ETHERNET_FRAME_SIZE - 2); + } + + // Try to release a packet that may need retransmission. + // + // If the release didn't happen, we'll be given back the + // packet through another send call. + packet.transmit_release(); + } + } + } +} |