add developper documentation (cargo doc)

README.md

@@ -13,4 +13,16 @@ For more information about the general purpose and concept of unidirectional net
 
 ## Where to find some documentation?
 
-The documentation is available at <https://anssi-fr.github.io/lidi/>.
+The *user* documentation is available at <https://anssi-fr.github.io/lidi/>, or can be built and opened with:
+
+```
+$ cd doc
+$ make html
+$ xdg-open _build/html/index.html
+```
+
+The *developper* documentation can be built and opened by running:
+
+```
+$ cargo doc --document-private-items --no-deps --lib --open
+```

src/file/mod.rs

@@ -1,3 +1,4 @@
+//! Module for sending/receiving entire files into/from Lidi TCP or Unix sockets
 pub mod protocol;
 pub mod receive;
 pub mod send;

src/protocol.rs

@@ -1,3 +1,37 @@
+//! Definition of the Lidi protocol used to transfer data over UDP
+//!
+//! The Lidi protocol is rather simple: since the communications are unidirectional, it is defined
+//! by the messages structure. There are 5 message types:
+//! - `MessageType::Heartbeat` lets know the receiver that transfer can happen,
+//! - `MessageType::Start` informs the receiver that the sent data chunk represents the beginning of a new transfer,
+//! - `MessageType::Data` is used to send a data chunk that is not the beginning nor the ending of
+//! a transfer,
+//! - `MessageType::Abort` informs the receiver that the current transfer has been aborted on the
+//! sender side,
+//! - `MessageType::End` informs the receiver that the current transfer is completed (i.e. all
+//! data have been sent).
+//!
+//! A message is stored in a `Vec` of `u8`s, with the following representation:
+//!
+//! ```text
+//!
+//! <-- 4 bytes -> <--- 1 byte ---> <-- 4 bytes -->
+//! --------------+----------------+---------------+--------------------------------------
+//! |             |                |               |                                     |
+//! |  client_id  |  message_type  |  data_length  |  payload = data + optional padding  |
+//! |             |                |               |                                     |
+//! --------------+----------------+---------------+--------------------------------------
+//!  <------------ SERIALIZE_OVERHEAD ------------> <--------- message_length ---------->
+//!
+//! ```
+//!
+//! 4-bytes values are encoded in little-endian byte order.
+//!
+//! In `Heartbeat` messages, `client_id` is unused and should be set to 0 by the constructor
+//! caller. Also no data payload should be provided by the constructor caller in case the message
+//! is of type `Heartbeat`, `Abort` or `End`. Then the `data_length` will be set to 0 by the
+//! message constructor and the data chunk will be fully padded with zeros.
+
 use std::{fmt, io};
 
 pub enum Error {
@@ -69,6 +103,14 @@ pub struct Message(Vec<u8>);
 const SERIALIZE_OVERHEAD: usize = 4 + 1 + 4;
 
 impl Message {
+    /// Message constructor, craft a message according to the representation introduced in
+    /// [crate::protocol].
+    ///
+    /// Some (unchecked) constraints on arguments must be respected:
+    /// - if `message` is `MessageType::Heartbeat`, `MessageType::Abort` or `MessageType::End`
+    /// then no data should be provided,
+    /// - if `message` is `MessageType::Heartbear` then `client_id` should be equal to 0,
+    /// - if there is some `data`, its length must be greater than `message_length`.
     pub(crate) fn new(
         message: MessageType,
         message_length: u32,
@@ -121,11 +163,11 @@ impl Message {
         u32::from_le_bytes(data_len_bytes)
     }
 
-    pub(crate) fn deserialize(data: Vec<u8>) -> Self {
+    pub(crate) const fn deserialize(data: Vec<u8>) -> Self {
         Self(data)
     }
 
-    pub fn serialize_overhead() -> usize {
+    pub const fn serialize_overhead() -> usize {
         SERIALIZE_OVERHEAD
     }
 

src/receive/client.rs

@@ -1,3 +1,5 @@
+//! Worker that writes decoded and reordered messages to client
+
 use crate::{protocol, receive, sock_utils};
 use std::{
     io::{self, Write},

src/receive/clients.rs

@@ -1,3 +1,5 @@
+//! Worker that aquires multiplex access and then becomes a `crate::receive::client` worker
+
 use crate::{receive, receive::client};
 use std::{io::Write, os::fd::AsRawFd};
 

src/receive/decoding.rs

@@ -1,3 +1,5 @@
+//! Worker that decodes RaptorQ packets into protocol messages
+
 use crate::{protocol, receive};
 
 pub(crate) fn start<F>(receiver: &receive::Receiver<F>) -> Result<(), receive::Error> {

src/receive/dispatch.rs

@@ -1,3 +1,6 @@
+//! Worker that manages active transfers queue and dispatch incoming [crate::protocol]
+//! messages to clients
+
 use crate::{protocol, receive};
 use std::{
     collections::{BTreeMap, BTreeSet},

src/receive/mod.rs

@@ -1,3 +1,21 @@
+//! Receiver functions module
+//!
+//! Several threads are involved in the receipt pipeline. Each worker is run with a `start`
+//! function of a submodule of the [crate::receive] module, data being passed through
+//! [crossbeam_channel] bounded channels to form the following data pipeline:
+//!
+//! ```text
+//!       -----------             ------------------               ------------
+//! udp --| packets |-> reblock --| vec of packets |-> decodings --| messages |-> dispatch
+//!       -----------             ------------------               ------------
+//! ```
+//!
+//! Notes:
+//! - heartbeat does not need a dedicated worker on the receiver side, heartbeat messages are
+//! handled by the dispatch worker,
+//! - there are `nb_clients` clients workers running in parallel,
+//! - there are `nb_decoding_threads` decoding workers running in parallel.
+
 use crate::{protocol, semaphore};
 use std::{
     fmt,
@@ -130,6 +148,8 @@ impl From<protocol::Error> for Error {
     }
 }
 
+/// An instance of this data structure is shared by workers to synchronize them and to access
+/// communication channels
 pub struct Receiver<F> {
     pub(crate) config: Config,
     pub(crate) object_transmission_info: raptorq::ObjectTransmissionInformation,

src/receive/reblock.rs

@@ -1,3 +1,6 @@
+//! Worker for grouping packets according to their block numbers to handle potential UDP packets
+//! reordering
+
 use crate::{protocol, receive};
 
 pub(crate) fn start<F>(receiver: &receive::Receiver<F>) -> Result<(), receive::Error> {

src/semaphore.rs

@@ -1,3 +1,5 @@
+//! Simple semaphores built on top of `std::sync` primitives, no external dependency
+
 use std::sync::{Arc, Condvar, Mutex};
 
 #[derive(Clone)]

src/send/client.rs

@@ -1,3 +1,5 @@
+//! Worker that reads data from a client socket and split it into [crate::protocol] messages
+
 use crate::{protocol, send, sock_utils};
 use std::{io, os::fd::AsRawFd};
 

src/send/encoding.rs

@@ -1,3 +1,5 @@
+//! Worker that encodes protocol messages into RaptorQ packets
+
 use crate::{protocol, send};
 
 pub(crate) fn start<C>(sender: &send::Sender<C>) -> Result<(), send::Error> {

src/send/heartbeat.rs

@@ -1,3 +1,5 @@
+//! Optional worker that periodically inserts [crate::protocol] heartbeat message in the encoding queue
+
 use crate::{protocol, send};
 
 pub(crate) fn start<C>(sender: &send::Sender<C>) -> Result<(), send::Error> {

src/send/mod.rs

@@ -1,3 +1,24 @@
+//! Sender functions module
+//!
+//! Several threads are used to form a pipeline for the data to be prepared before sending it over
+//! UDP. Every submodule of the [crate::send] module is equipped with a `start` function that
+//! launch the worker process. Data pass through the workers pipelines via [crossbeam_channel]
+//! bounded channels.
+//!
+//! Here follows a simplified representation of the workers pipeline:
+//!
+//! ```text
+//!             ----------             ------------               -----------
+//! listeners --| client |-> clients --| messages |-> encodings --| packets |-> udp
+//!             ----------             ------------               -----------
+//! ```
+//!
+//! Notes:
+//! - listeners threads are spawned from binary and not the library crate,
+//! - heartbeat worker has been omitted from the representation for readability,
+//! - there are `nb_clients` clients workers running in parallel,
+//! - there are `nb_encoding_threads` encoding workers running in parallel.
+
 use crate::{protocol, semaphore};
 use std::{
     fmt,
@@ -89,6 +110,11 @@ impl From<protocol::Error> for Error {
     }
 }
 
+/// An instance of this data structure is shared by workers to synchronize them and to access
+/// communication channels
+///
+/// The `C` type variable represents the socket from which data is read before being sent over the
+/// diode.
 pub struct Sender<C> {
     pub(crate) config: Config,
     pub(crate) object_transmission_info: raptorq::ObjectTransmissionInformation,

src/send/server.rs

@@ -1,3 +1,5 @@
+//! Worker that gets a client socket and becomes a `crate::send::client` worker
+
 use crate::{protocol, send, send::client};
 use std::{io::Read, os::fd::AsRawFd};
 

src/sock_utils.rs

@@ -1,3 +1,5 @@
+//! Bindings and wrappers for socket buffer size libc funtions
+
 use std::os::fd::AsRawFd;
 use std::{io, mem, ptr};
 

src/udp.rs

@@ -1,10 +1,17 @@
+//! Functions and wrappers over libc's UDP socket multiple messages receive and send
+
 use std::marker::PhantomData;
 use std::os::fd::AsRawFd;
 use std::{io, mem, net};
 
 pub struct UdpRecv;
 pub struct UdpSend;
 
+/// Wrapper structure over the socket and buffers used to send and receive multiple messages.
+/// Inner data are used to call libc recvmmsg and sendmmsg.
+///
+/// The `D` type parameter is intended to be [UdpRecv] or [UdpSend] to ensure structures are
+/// correctly initialized according to the data transfer direction.
 pub struct UdpMessages<D> {
     socket: net::UdpSocket,
     vlen: usize,