6 files changed, 88 insertions, 14 deletions

diff --git a/rtl/gfx/fold_flow.sv b/rtl/gfx/fold_flow.sv
index 718786e..1d9c0f2 100644
--- a/rtl/gfx/fold_flow.sv
+++ b/rtl/gfx/fold_flow.sv
@@ -15,17 +15,23 @@ module fold_flow
 	              feedback_last
 );
 
+	logic skid_ready;
 	index4 rounds[`FP_ADD_STAGES], last_round;
 
-	assign stall = out_valid && !out_ready;
-	assign in_ready = !stall && !feedback;
-	assign out_valid = last_round == `INDEX4_MAX;
+	assign in_ready = skid_ready && !feedback;
 
 	assign feedback = last_round[1] ^ last_round[0];
 	assign feedback_last = last_round[1];
 
 	assign last_round = rounds[`FP_ADD_STAGES - 1];
 
+	skid_flow skid
+	(
+		.in_valid(last_round == `INDEX4_MAX),
+		.in_ready(skid_ready),
+		.*
+	);
+
 	always_ff @(posedge clk or negedge rst_n)
 		if (!rst_n)
 			rounds[0] <= `INDEX4_MIN;
diff --git a/rtl/gfx/horizontal_fold.sv b/rtl/gfx/horizontal_fold.sv
index 513e3b1..ee56098 100644
--- a/rtl/gfx/horizontal_fold.sv
+++ b/rtl/gfx/horizontal_fold.sv
@@ -12,14 +12,23 @@ module horizontal_fold
 	output fp     q
 );
 
+	fp q_add;
 	vec2 feedback_vec, queued[`FP_ADD_STAGES];
 
 	assign feedback_vec = queued[`FP_ADD_STAGES - 1];
 
 	fp_add add
 	(
-		.a(feedback ? q : vec[0]),
+		.a(feedback ? q_add : vec[0]),
 		.b(feedback ? feedback_vec[feedback_last] : vec[1]),
+		.q(q_add),
+		.*
+	);
+
+	skid_buf #(.WIDTH($bits(q))) skid
+	(
+		.in(q_add),
+		.out(q),
 		.*
 	);
 
diff --git a/rtl/gfx/pipeline_flow.sv b/rtl/gfx/pipeline_flow.sv
index 2b9c891..15986d0 100644
--- a/rtl/gfx/pipeline_flow.sv
+++ b/rtl/gfx/pipeline_flow.sv
@@ -1,5 +1,3 @@
-`include "gfx/gfx_defs.sv"
-
 module pipeline_flow
 #(parameter STAGES=0)
 (
@@ -16,14 +14,16 @@ module pipeline_flow
 
 	logic valid[STAGES];
 
-	assign stall = !in_ready;
-	assign in_ready = out_ready || !out_valid;
-	assign out_valid = valid[STAGES - 1];
+	skid_flow skid
+	(
+		.in_valid(valid[STAGES - 1]),
+		.*
+	);
 
 	always_ff @(posedge clk or negedge rst_n)
 		if (!rst_n)
 			valid[0] <= 0;
-		else if (in_ready)
+		else if (!stall)
 			valid[0] <= in_valid;
 
 	genvar i;
@@ -32,7 +32,7 @@ module pipeline_flow
 			always_ff @(posedge clk or negedge rst_n)
 				if (!rst_n)
 					valid[i] <= 0;
-				else if (in_ready)
+				else if (!stall)
 					valid[i] <= valid[i - 1];
 		end
 	endgenerate
diff --git a/rtl/gfx/skid_buf.sv b/rtl/gfx/skid_buf.sv
new file mode 100644
index 0000000..6e7ffbb
--- /dev/null
+++ b/rtl/gfx/skid_buf.sv
@@ -0,0 +1,20 @@
+module skid_buf
+#(parameter WIDTH=0)
+(
+	input  logic              clk,
+
+	input  logic[WIDTH - 1:0] in,
+	input  logic              stall,
+
+	output logic[WIDTH - 1:0] out
+);
+
+	logic[WIDTH - 1:0] skid;
+
+	assign out = stall ? skid : in;
+
+	always_ff @(posedge clk)
+		if (!stall)
+			skid <= in;
+
+endmodule
diff --git a/rtl/gfx/skid_flow.sv b/rtl/gfx/skid_flow.sv
new file mode 100644
index 0000000..a38df65
--- /dev/null
+++ b/rtl/gfx/skid_flow.sv
@@ -0,0 +1,31 @@
+module skid_flow
+(
+	input  logic clk,
+	             rst_n,
+
+	input  logic in_valid,
+	             out_ready,
+
+	output logic in_ready,
+	             out_valid,
+	             stall
+);
+
+	logic was_ready, was_valid;
+
+	assign stall = !in_ready;
+	assign in_ready = was_ready || !was_valid;
+	assign out_valid = in_valid || was_valid;
+
+	always @(posedge clk or negedge rst_n)
+		if (!rst_n) begin
+			was_ready <= 0;
+			was_valid <= 0;
+		end else begin
+			was_ready <= out_ready;
+
+			if (!stall)
+				was_valid <= in_valid;
+		end
+
+endmodule
diff --git a/rtl/gfx/vec_dot.sv b/rtl/gfx/vec_dot.sv
index 4e1fdee..c0926ac 100644
--- a/rtl/gfx/vec_dot.sv
+++ b/rtl/gfx/vec_dot.sv
@@ -15,11 +15,11 @@ module vec_dot
 	output fp    q
 );
 
-	vec4 products;
+	vec4 products_fold, products_mul;
 
 	horizontal_fold fold
 	(
-		.vec(products),
+		.vec(products_fold),
 		.stall(stall_fold),
 		.*
 	);
@@ -31,7 +31,15 @@ module vec_dot
 			(
 				.a(a[i]),
 				.b(b[i]),
-				.q(products[i]),
+				.q(products_mul[i]),
+				.stall(stall_mul),
+				.*
+			);
+
+			skid_buf #(.WIDTH($bits(vec4))) skid_i
+			(
+				.in(products_mul[i]),
+				.out(products_fold[i]),
 				.stall(stall_mul),
 				.*
 			);