idma: calibrate TCDM↔AXI transfer timing to match RTL on magia_v2

pulp/chips/magia_v2/arch.py

@@ -96,7 +96,7 @@ class MagiaDSE:
     TILE_AXI_XBAR_SYNC          = False
     TILE_OBI_XBAR_LATENCY       = 2
     TILE_OBI_XBAR_SYNC          = True
-    TILE_IDMA0_BQUEUE_SIZE      = 2
-    TILE_IDMA0_B_SIZE           = 32
-    TILE_IDMA1_BQUEUE_SIZE      = 2
-    TILE_IDMA1_B_SIZE           = 32
+    TILE_IDMA0_BQUEUE_SIZE      = 4
+    TILE_IDMA0_B_SIZE           = 0
+    TILE_IDMA1_BQUEUE_SIZE      = 4
+    TILE_IDMA1_B_SIZE           = 0

pulp/floonoc/floonoc.hpp

@@ -100,7 +100,8 @@ class FlooNoc : public vp::Component
     static constexpr int REQ_DEST_Y = 3;      // Y coordinate of the destination target
     static constexpr int REQ_WIDE = 4;        // Indicates if a request is a wide request or not. 1 for wide, 0 for narrow
     static constexpr int REQ_IS_ADDRESS = 5;     // Indicates if the request is a AR/AW request or not. 1 for address, 0 for data
-    static constexpr int REQ_NB_ARGS = 6;     // Number of request data required by this model
+    static constexpr int REQ_IS_LAST = 6;     // Set on the last beat of a burst. Used by the routers to implement wormhole arbitration (a burst keeps its output port locked until its last beat went through)
+    static constexpr int REQ_NB_ARGS = 7;     // Number of request data required by this model
 
     // The following constants gives the index in the queue array of the queue associated to each direction
     static constexpr int DIR_RIGHT = 0;

pulp/floonoc/floonoc_network_interface.cpp

@@ -168,6 +168,11 @@ void NetworkQueue::enqueue_router_req(vp::IoReq *req, bool is_address, bool wide
             *router_req->arg_get(FlooNoc::REQ_BURST) = *req->arg_get(FlooNoc::REQ_BURST);
         }
 
+        // Mark the last beat of the burst so that the routers know when to release the output
+        // port they locked for this burst (wormhole arbitration, see Router::fsm_handler).
+        bool is_last = (burst_size - size == 0);
+        *router_req->arg_get(FlooNoc::REQ_IS_LAST) = (void *)(long)is_last;
+
         this->queue.push(router_req);
 
         burst_base += size;

pulp/floonoc/floonoc_router.cpp

@@ -140,6 +140,24 @@ void Router::fsm_handler(vp::Block *__this, vp::ClockEvent *event)
             // Get output queue ID from next position
             int out_queue_id = _this->get_req_queue(next_x, next_y);
 
+            // Wormhole arbitration: if this output is locked to another input (a multi-beat burst
+            // is still in flight through it), this input must wait until that burst's last beat
+            // has gone through. This reserves the output for the whole burst, exactly like the RTL
+            // floo_wormhole_arbiter (LockIn=1, valid resampled only at packet boundaries). It is
+            // what gives the closest injector near-full bandwidth: the far traffic holds long
+            // multi-hop paths and arrives in bursts, so the local input wins the idle slots.
+            if (_this->locked_output[out_queue_id] != -1 &&
+                _this->locked_output[out_queue_id] != in_queue_index)
+            {
+                _this->fsm_event.enqueue(); // Retry once the burst releases the output
+                in_queue_index += 1;
+                if (in_queue_index == 5)
+                {
+                    in_queue_index = 0;
+                }
+                continue;
+            }
+
             // Only send one request per cycle to the same output
             if (output_full[out_queue_id])
             {
@@ -171,6 +189,11 @@ void Router::fsm_handler(vp::Block *__this, vp::ClockEvent *event)
             // Since we now know, that the request will be propagated, remove it from the queue
             queue->queue.pop();
 
+            // Update the wormhole lock for this output: keep it reserved for this input until the
+            // burst's last beat goes through, then release it so other inputs can compete again.
+            bool is_last = (bool)(long)*req->arg_get(FlooNoc::REQ_IS_LAST);
+            _this->locked_output[out_queue_id] = is_last ? -1 : in_queue_index;
+
             if (queue->queue.size() == _this->queue_size) // Remember we let the source enqueue one more request than what is possible.
             {
                 // In case the queue had one more element than possible, it means the output
@@ -330,6 +353,7 @@ void Router::reset(bool active)
         for (int i = 0; i < 5; i++)
         {
             this->stalled_queues[i] = false;
+            this->locked_output[i] = -1;
         }
     }
 

pulp/floonoc/floonoc_router.hpp

@@ -91,6 +91,11 @@ class Router : public FloonocNode
     vp::ClockEvent fsm_event;
     // Current queue where next request will be taken from, used for round-robin
     int current_queue;
+    // Wormhole arbitration state: for each output direction, the input queue index that currently
+    // owns it (a multi-beat burst is in flight through that output), or -1 if the output is free.
+    // While an output is locked, only the owning input may use it, until the burst's last beat
+    // has passed. This mirrors the RTL floo_wormhole_arbiter (LockIn=1).
+    int locked_output[5];
     // State of the output queues, true if it is stalled and nothing can be sent to it anymore
     // until it is unstalled.
     std::array<vp::Signal<bool>, 5> stalled_queues;

pulp/idma/be/idma_be.cpp

@@ -202,6 +202,34 @@ void IDmaBe::ack_data(IdmaTransfer *transfer, uint8_t *data, int size)
 
 
 
+// Release a source buffer for flow control, without accounting it as written. Used by
+// destination back-ends that report completion later (see ack_write_completed).
+void IDmaBe::ack_data_buffer(IdmaTransfer *transfer, uint8_t *data)
+{
+    IdmaBeConsumer *src_be = this->get_be_consumer(transfer->src, transfer->size, true);
+    src_be->write_data_ack(data);
+}
+
+
+
+// Account data as actually written and terminate the transfer once it is fully committed.
+// Counterpart of ack_data_buffer, it does not touch any source buffer.
+void IDmaBe::ack_write_completed(IdmaTransfer *transfer, int size)
+{
+    this->trace.msg(vp::Trace::LEVEL_TRACE, "Acknowledging written (size: 0x%x, remaining_size: 0x%x)\n",
+        size, transfer->ack_size);
+
+    transfer->ack_size -= size;
+
+    if (transfer->ack_size == 0)
+    {
+        this->trace.msg(vp::Trace::LEVEL_TRACE, "Finished burst (transfer: %p)\n", transfer);
+        this->me->ack_transfer(transfer);
+    }
+}
+
+
+
 void IDmaBe::reset(bool active)
 {
     if (active)

pulp/idma/be/idma_be.hpp

@@ -196,6 +196,34 @@ class IdmaBeProducer
      * @param size Size of the written data being acknowledged
      */
     virtual void ack_data(IdmaTransfer *transfer, uint8_t *data, int size) = 0;
+
+    /**
+     * @brief Release a source data buffer without accounting it as written
+     *
+     * This splits the two roles of ack_data for back-ends that decouple the moment a source
+     * buffer can be freed (flow control) from the moment the data is actually committed to the
+     * destination (transfer completion). It only releases the source buffer so the source can
+     * keep streaming; it does NOT decrement the remaining transfer size. The destination must
+     * later call ack_write_completed when the data has really been written.
+     *
+     * @param transfer Transfer for which the source buffer is released
+     * @param data Source data buffer to release
+     */
+    virtual void ack_data_buffer(IdmaTransfer *transfer, uint8_t *data) = 0;
+
+    /**
+     * @brief Account data as actually written and possibly terminate the transfer
+     *
+     * Counterpart of ack_data_buffer: it accounts the given amount of data as committed to the
+     * destination and terminates the transfer once everything has been written. It does not
+     * touch any source buffer (that is done earlier through ack_data_buffer). This lets a
+     * back-end report completion at the real destination-write time (e.g. a NoC write response)
+     * rather than at source-read time.
+     *
+     * @param transfer Transfer for which written data is accounted
+     * @param size Size of the written data being accounted
+     */
+    virtual void ack_write_completed(IdmaTransfer *transfer, int size) = 0;
 };
 
 
@@ -232,6 +260,8 @@ class IDmaBe : public vp::Block, public IdmaTransferConsumer, public IdmaBeProdu
     bool is_ready_to_accept_data(IdmaTransfer *transfer) override;
     void write_data(IdmaTransfer *transfer, uint8_t *data, uint64_t size) override;
     void ack_data(IdmaTransfer *transfer, uint8_t *data, int size) override;
+    void ack_data_buffer(IdmaTransfer *transfer, uint8_t *data) override;
+    void ack_write_completed(IdmaTransfer *transfer, int size) override;
 
 private:
     // FSM handler, called to check if any action should be taken after something was updated