Implement hardware virtual memory

13 files changed, 431 insertions, 24 deletions

diff --git a/conspiracion.qsf b/conspiracion.qsf
index e026a5d..4389fe5 100644
--- a/conspiracion.qsf
+++ b/conspiracion.qsf
@@ -254,6 +254,7 @@ set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/fetch/prefetch.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/mmu/arbiter.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/mmu/format.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/mmu/mmu.sv
+set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/mmu/pagewalk.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/mul.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/porch/conds.sv
 set_global_assignment -name SYSTEMVERILOG_FILE rtl/core/porch/porch.sv
diff --git a/rtl/core/mmu/format.sv b/rtl/core/mmu/format.sv
index c0918a3..83ca3eb 100644
--- a/rtl/core/mmu/format.sv
+++ b/rtl/core/mmu/format.sv
@@ -8,13 +8,66 @@ typedef logic[17:0] mmu_base;
 `define MMU_L1_PAGETABLE 2'b01
 `define MMU_L1_SECTION   2'b10
 
+typedef struct packed
+{
+	logic[31:10] base;
+	logic[9:9]   imp;
+	logic[8:5]   domain;
+	logic[4:2]   sbz;
+	logic[1:0]   typ;
+} mmu_l1_pagetable;
+
+typedef struct packed
+{
+	logic[31:20] base;
+	logic[19:15] sbz0;
+	logic[14:12] tex;
+	logic[11:10] ap;
+	logic[9:9]   imp;
+	logic[8:5]   domain;
+	logic[4:4]   sbz1;
+	logic[3:3]   c;
+	logic[2:2]   b;
+	logic[1:0]   typ;
+} mmu_l1_section;
+
+`define MMU_SECTION_INDEX [17:0]
+
 `define MMU_L2_INDEX    [17:10]
 `define MMU_L2_FAULT    2'b00
 `define MMU_L2_LARGE    2'b01
 `define MMU_L2_SMALL    2'b10
 `define MMU_L2_SMALLEXT 2'b11
 
+typedef struct packed
+{
+	logic[31:16] base;
+	logic[15:15] sbz;
+	logic[14:12] tex;
+	logic[11:10] ap3;
+	logic[9:8]   ap2;
+	logic[7:6]   ap1;
+	logic[5:4]   ap0;
+	logic[3:3]   c;
+	logic[2:2]   b;
+	logic[1:0]   typ;
+} mmu_l2_large;
+
+typedef struct packed
+{
+	logic[31:12] base;
+	logic[11:10] ap3;
+	logic[9:8]   ap2;
+	logic[7:6]   ap1;
+	logic[5:4]   ap0;
+	logic[3:3]   c;
+	logic[2:2]   b;
+	logic[1:0]   typ;
+} mmu_l2_small;
+
+//TODO: struct mmu_l2_smallext
+
 `define MMU_LARGE_INDEX [13:0]
-`define MMU_SMALL_INDEX [11:0]
+`define MMU_SMALL_INDEX [9:0]
 
 `endif
diff --git a/rtl/core/mmu/mmu.sv b/rtl/core/mmu/mmu.sv
index a909537..504e447 100644
--- a/rtl/core/mmu/mmu.sv
+++ b/rtl/core/mmu/mmu.sv
@@ -1,8 +1,14 @@
+`include "core/mmu/format.sv"
+`include "core/uarch.sv"
+
 module core_mmu
 (
 	input  logic      clk,
 	                  rst_n,
 
+	input  logic      mmu_enable /*verilator public*/,
+	input  mmu_base   mmu_ttbr /*verilator public*/,
+
 	input  logic      bus_ready,
 	input  word       bus_data_rd,
 	                  data_data_wr,
@@ -24,9 +30,68 @@ module core_mmu
 	                  data_data_rd
 );
 
-	//TODO
+	ptr iphys_addr, dphys_addr;
+	word iphys_data_rd, dphys_data_rd, dphys_data_wr;
+	logic iphys_start, dphys_start, iphys_ready, dphys_ready, dphys_write;
+	logic[3:0] dphys_data_be;
+
+	core_mmu_pagewalk iwalk
+	(
+		.core_addr(insn_addr),
+		.core_start(insn_start),
+		.core_write(0),
+		.core_ready(insn_ready),
+		.core_data_wr(0),
+		.core_data_be(0),
+		.core_data_rd(insn_data_rd),
+
+		.bus_addr(iphys_addr),
+		.bus_start(iphys_start),
+		.bus_write(),
+		.bus_ready(iphys_ready),
+		.bus_data_wr(),
+		.bus_data_be(),
+		.bus_data_rd(iphys_data_rd),
+
+		.*
+	);
+
+	core_mmu_pagewalk dwalk
+	(
+		.core_addr(data_addr),
+		.core_start(data_start),
+		.core_write(data_write),
+		.core_ready(data_ready),
+		.core_data_wr(data_data_wr),
+		.core_data_be(data_data_be),
+		.core_data_rd(data_data_rd),
+
+		.bus_addr(dphys_addr),
+		.bus_start(dphys_start),
+		.bus_write(dphys_write),
+		.bus_ready(dphys_ready),
+		.bus_data_wr(dphys_data_wr),
+		.bus_data_be(dphys_data_be),
+		.bus_data_rd(dphys_data_rd),
+
+		.*
+	);
+
 	core_mmu_arbiter arbiter
 	(
+		.insn_addr(iphys_addr),
+		.insn_start(iphys_start),
+		.insn_ready(iphys_ready),
+		.insn_data_rd(iphys_data_rd),
+
+		.data_addr(dphys_addr),
+		.data_start(dphys_start),
+		.data_write(dphys_write),
+		.data_ready(dphys_ready),
+		.data_data_wr(dphys_data_wr),
+		.data_data_be(dphys_data_be),
+		.data_data_rd(dphys_data_rd),
+
 		.*
 	);
 
diff --git a/rtl/core/mmu/pagewalk.sv b/rtl/core/mmu/pagewalk.sv
new file mode 100644
index 0000000..b16ce26
--- /dev/null
+++ b/rtl/core/mmu/pagewalk.sv
@@ -0,0 +1,165 @@
+`include "core/mmu/format.sv"
+`include "core/uarch.sv"
+
+module core_mmu_pagewalk
+(
+	input  logic      clk,
+	                  rst_n,
+
+	input  logic      mmu_enable,
+	input  mmu_base   mmu_ttbr,
+
+	input  logic      bus_ready,
+	input  word       bus_data_rd,
+
+	input  ptr        core_addr,
+	input  word       core_data_wr,
+	input  logic[3:0] core_data_be,
+	input  logic      core_start,
+	                  core_write,
+
+	output ptr        bus_addr,
+	output word       bus_data_wr,
+	output logic[3:0] bus_data_be,
+	output logic      bus_start,
+	                  bus_write,
+
+	output logic      core_ready,
+	output word       core_data_rd
+);
+
+	enum int unsigned
+	{
+		IDLE,
+		L1,
+		L2,
+		DATA,
+		FAULT
+	} state;
+
+	mmu_l1_pagetable pagetable;
+	assign pagetable = bus_data_rd;
+
+	mmu_l1_section section;
+	assign section = bus_data_rd;
+
+	mmu_l2_large ptentry_large;
+	assign ptentry_large = bus_data_rd;
+
+	mmu_l2_small ptentry_small;
+	assign ptentry_small = bus_data_rd;
+
+	ptr target;
+	word hold_data;
+	logic[3:0] hold_be;
+
+	logic hold_write;
+
+	always @(posedge clk or negedge rst_n)
+		if(!rst_n) begin
+			state <= IDLE;
+			target <= 0;
+
+			hold_be <= 0;
+			hold_data <= 0;
+			hold_write <= 0;
+
+			bus_addr <= 0;
+			bus_start <= 0;
+			bus_write <= 0;
+			bus_data_be <= 0;
+			bus_data_wr <= 0;
+
+			core_ready <= 0;
+			core_data_rd <= 0;
+		end else begin
+			if(bus_start)
+				bus_start <= 0;
+
+			if(core_ready)
+				core_ready <= 0;
+
+			unique case(state)
+				IDLE:
+					if(core_start) begin
+						bus_start <= 1;
+
+						if(mmu_enable) begin
+							target <= core_addr;
+							hold_be <= core_data_be;
+							hold_data <= core_data_wr;
+							hold_write <= core_write;
+
+							state <= L1;
+							bus_addr <= {mmu_ttbr, core_addr `MMU_L1_INDEX};
+						end else begin
+							state <= DATA;
+							bus_addr <= core_addr;
+							bus_write <= core_write;
+							bus_data_wr <= core_data_wr;
+							bus_data_be <= core_data_be;
+						end
+					end
+
+				L1:
+					if(bus_ready)
+						unique case(bus_data_rd[1:0])
+							`MMU_L1_PAGETABLE: begin
+								state <= L2;
+								bus_addr <= {pagetable.base, target `MMU_L2_INDEX};
+								bus_start <= 1;
+							end
+
+							`MMU_L1_SECTION: begin
+								state <= DATA;
+
+								bus_addr <= {section.base, target `MMU_SECTION_INDEX};
+								bus_start <= 1;
+								bus_write <= hold_write;
+								bus_data_wr <= hold_data;
+								bus_data_be <= hold_be;
+							end
+
+							// Tiny (1KiB wtf?) pages and supersections not supported
+							default:
+								state <= FAULT;
+						endcase
+
+				L2:
+					if(bus_ready) begin
+						state <= DATA;
+
+						bus_write <= hold_write;
+						bus_data_wr <= hold_data;
+						bus_data_be <= hold_be;
+
+						unique case(bus_data_rd[1:0])
+							`MMU_L2_FAULT:
+								state <= FAULT;
+
+							`MMU_L2_LARGE: begin
+								bus_addr <= {ptentry_large.base, target `MMU_LARGE_INDEX};
+								bus_start <= 1;
+							end
+
+							`MMU_L2_SMALL, `MMU_L2_SMALLEXT: begin
+								bus_addr <= {ptentry_small.base, target `MMU_SMALL_INDEX};
+								bus_start <= 1;
+							end
+						endcase
+					end
+
+				DATA:
+					if(bus_ready) begin
+						state <= IDLE;
+						bus_write <= 0;
+						core_ready <= 1;
+						core_data_rd <= bus_data_rd;
+					end
+
+				//TODO
+				FAULT: ;
+			endcase
+		end
+
+endmodule
diff --git a/sim/gdbstub.py b/sim/gdbstub.py
index 914f717..74d046c 100644
--- a/sim/gdbstub.py
+++ b/sim/gdbstub.py
@@ -86,7 +86,7 @@ def yield_to_gdb():
             out = hexout(read_reg(gdb_reg(r)) for r in range(16))
         elif data[0] == b'm'[0]:
             addr, length = (int(x, 16) for x in data[1:].split(b','))
-            out = hexout(read_mem(addr, length))
+            out = hexout(read_mem(addr, length, may_fail = True))
         elif data[0] == b'M'[0]:
             addrlen, data = data[1:].split(b':')
             addr, length = (int(x, 16) for x in addrlen.split(b','))
diff --git a/sim/sim.py b/sim/sim.py
index 491ac43..b4e0882 100755
--- a/sim/sim.py
+++ b/sim/sim.py
@@ -85,10 +85,14 @@ def recv_mem_dump():
         elif line == '=== end-mem ===':
             break
 
-        base, data = line.split()
-        dumped.append((int(base, 16) << 2, bytes.fromhex(data)))
+        try:
+            base, data = line.split()
+            dumped.append((int(base, 16) << 2, bytes.fromhex(data)))
+        except ValueError:
+            while_running()
+            print(f'{COLOR_BLUE}{line}{COLOR_RESET}')
 
-def read_mem(base, length):
+def read_mem(base, length, *, may_fail = False):
     fragments = []
     i = 0
 
@@ -97,6 +101,9 @@ def read_mem(base, length):
         recv_mem_dump()
 
     while length > 0:
+        if i >= len(dumped) and may_fail:
+            return None
+
         assert i < len(dumped), f'memory at 0x{base:08x} not dumped'
         start, data = dumped[i]
         delta = base - start
diff --git a/tb/avalon.hpp b/tb/avalon.hpp
index f37b306..763065c 100644
--- a/tb/avalon.hpp
+++ b/tb/avalon.hpp
@@ -80,7 +80,7 @@ namespace taller::avalon
 			void attach(slave &dev);
 			void bail() noexcept;
 
-			std::uint32_t dump(std::uint32_t addr);
+			bool dump(std::uint32_t addr, std::uint32_t &word);
 			void patch(std::uint32_t addr, std::uint32_t readdata);
 
 		private:
@@ -100,7 +100,7 @@ namespace taller::avalon
 			bool                 avl_read       = false;
 			bool                 avl_write      = false;
 
-			slave &resolve_external(std::uint32_t avl_address);
+			slave *resolve_external(std::uint32_t avl_address);
 	};
 }
 
diff --git a/tb/avalon.impl.hpp b/tb/avalon.impl.hpp
index 3af60d0..4dee4f2 100644
--- a/tb/avalon.impl.hpp
+++ b/tb/avalon.impl.hpp
@@ -125,49 +125,55 @@ namespace taller::avalon
 	}
 
 	template<class Platform>
-	std::uint32_t interconnect<Platform>::dump(std::uint32_t addr)
+	bool interconnect<Platform>::dump(std::uint32_t addr, std::uint32_t &word)
 	{
 		std::uint32_t avl_address = addr << 2;
-		auto &dev = resolve_external(avl_address);
 
-		auto pos = (avl_address & ~dev.address_mask()) >> dev.word_bits();
+		auto *dev = resolve_external(avl_address);
+		if(!dev)
+		{
+			return false;
+		}
+
+		auto pos = (avl_address & ~dev->address_mask()) >> dev->word_bits();
 
-		std::uint32_t readdata;
-		while(!dev.read(pos, readdata))
+		while(!dev->read(pos, word))
 		{
 			continue;
 		}
 
-		return readdata;
+		return true;
 	}
 
 	template<class Platform>
 	void interconnect<Platform>::patch(std::uint32_t addr, std::uint32_t writedata)
 	{
 		std::uint32_t avl_address = addr << 2;
-		auto &dev = resolve_external(avl_address);
 
-		auto pos = (avl_address & ~dev.address_mask()) >> dev.word_bits();
+		auto *dev = resolve_external(avl_address);
+		assert(dev);
+
+		auto pos = (avl_address & ~dev->address_mask()) >> dev->word_bits();
 
-		while(!dev.write(pos, writedata, 0b1111))
+		while(!dev->write(pos, writedata, 0b1111))
 		{
 			continue;
 		}
 	}
 
 	template<class Platform>
-	slave& interconnect<Platform>::resolve_external(std::uint32_t avl_address)
+	slave* interconnect<Platform>::resolve_external(std::uint32_t avl_address)
 	{
 		for(auto &binding : devices)
 		{
 			if((avl_address & binding.mask) == binding.base)
 			{
-				return binding.dev;
+				return &binding.dev;
 			}
 		}
 
 		fprintf(stderr, "[avl] attempt to access hole at 0x%08x\n", avl_address);
-		assert(false);
+		return nullptr;
 	}
 }
 
diff --git a/tb/mem.impl.hpp b/tb/mem.impl.hpp
index f7bb424..e3c11e8 100644
--- a/tb/mem.impl.hpp
+++ b/tb/mem.impl.hpp
@@ -38,7 +38,7 @@ namespace taller::avalon
 	bool mem<Cell>::read(std::uint32_t addr, std::uint32_t &data)
 	{
 		data = block[addr];
-		return ready();
+		return true;/*ready();*/
 	}
 
 	template<typename Cell>
@@ -67,7 +67,7 @@ namespace taller::avalon
 		}
 
 		block[addr] = (data & bytes) | (block[addr] & ~bytes);
-		return ready();
+		return true;/*ready();*/
 	}
 
 	template<typename Cell>
diff --git a/tb/sim/descifrador.py b/tb/sim/descifrador.py
index 5d81686..e754f2a 100644
--- a/tb/sim/descifrador.py
+++ b/tb/sim/descifrador.py
@@ -5,7 +5,7 @@ START = 0x10000
 
 loads = {START: FILE}
 consts = {0x30050000: 1, 0x30060000: 0}
-cycles = 20000000
+cycles = 23000000
 mem_dumps = [range(START, START + SIZE)]
 
 def final():
diff --git a/tb/sim/paging.S b/tb/sim/paging.S
new file mode 100644
index 0000000..a2c1d14
--- /dev/null
+++ b/tb/sim/paging.S
@@ -0,0 +1,42 @@
+.global reset
+reset:
+	# Copy code to page 3
+	ldr  r0, =virtual_start
+	ldr  r1, =0x3000
+	ldr  r2, =256
+	.copy_virtual:
+	ldr  r3, [r0], #4
+	str  r3, [r1], #4
+	subs r2, r2, #4
+	bne  .copy_virtual
+
+	# Translation tables
+	mov r0, #0x4000
+	ldr r1, =0x00005001
+	str r1, [r0]
+
+	mov r1, #0x5000
+	ldr r2, =0x00000002
+	str r2, [r1], #4
+	str r2, [r1], #4
+	str r2, [r1], #4
+	str r2, [r1], #4
+	ldr r2, =0x00003002
+	str r2, [r1], #4
+
+	# Set translation base and enable MMU
+	mov r1, #(1 << 0)
+	mcr p15, 0, r0, c2, c0, 0
+	mcr p15, 0, r1, c1, c0, 0
+
+	# Self-relocate to 0x2000 (mirror of 0x0000)
+	ldr r1, =#(0x2000 - 4)
+	add pc, pc, r1
+
+	# Jump to virtual_start (phys: 0x3000, virt: 0x4000)
+	ldr r0, =0x01234567
+	ldr pc, =0x4000
+
+virtual_start:
+	ldr r1, =0x89abcdef
+	mov pc, lr
diff --git a/tb/sim/paging.py b/tb/sim/paging.py
new file mode 100644
index 0000000..799f03d
--- /dev/null
+++ b/tb/sim/paging.py
@@ -0,0 +1,5 @@
+cycles = 4096
+
+def final():
+    assert_reg(r0, 0x01234567)
+    assert_reg(r1, 0x89abcdef)
diff --git a/tb/top/conspiracion.cpp b/tb/top/conspiracion.cpp
index 099baf3..b7ec3eb 100644
--- a/tb/top/conspiracion.cpp
+++ b/tb/top/conspiracion.cpp
@@ -17,6 +17,7 @@
 #include "Vconspiracion_platform.h"
 #include "Vconspiracion_vga_domain.h"
 #include "Vconspiracion_core_control.h"
+#include "Vconspiracion_core_mmu.h"
 #include "Vconspiracion_core_psr.h"
 #include "Vconspiracion_core_regs.h"
 #include "Vconspiracion_core_reg_file.h"
@@ -431,6 +432,57 @@ int main(int argc, char **argv)
 
 	auto do_mem_dump = [&](const mem_region *dumps, std::size_t count)
 	{
+		bool mmu_enabled = top.conspiracion->core->mmu->mmu_enable;
+		std::uint32_t ttbr = top.conspiracion->core->mmu->mmu_ttbr;
+
+		auto pagewalk = [&](std::uint32_t &addr)
+		{
+			if(!mmu_enabled)
+			{
+				return true;
+			}
+
+			std::uint32_t entry;
+			if(!avl.dump(ttbr << 12 | addr >> 18, entry))
+			{
+				return false;
+			}
+
+			switch(entry & 0b11)
+			{
+				case 0b01:
+					break;
+
+				case 0b10:
+					addr = (entry & ~((1 << 20) - 1)) >> 2 | (addr & ((1 << 18) - 1));
+					return true;
+
+				default:
+					return false;
+			}
+
+			std::uint32_t entryaddr = (entry & ~((1 << 10) - 1)) >> 2 | ((addr >> 10) & ((1 << 8) - 1));
+			if(!avl.dump(entryaddr, entry))
+			{
+				return false;
+			}
+
+			switch(entry & 0b11)
+			{
+				case 0b01:
+					addr = (entry & ~((1 << 16) - 1)) >> 2 | (addr & ((1 << 14) - 1));
+					return true;
+
+				case 0b10:
+				case 0b11:
+					addr = (entry & ~((1 << 12) - 1)) >> 2 | (addr & ((1 << 10) - 1));
+					return true;
+
+				default:
+					return false;
+			}
+		};
+
 		std::fputs("=== dump-mem ===\n", ctrl);
 		for(std::size_t i = 0; i < count; ++i)
 		{
@@ -439,7 +491,18 @@ int main(int argc, char **argv)
 			std::fprintf(ctrl, "%08x ", static_cast<std::uint32_t>(dump.start));
 			for(std::size_t i = 0; i < dump.length; ++i)
 			{
-				auto word = avl.dump(dump.start + i);
+				std::uint32_t at = dump.start + i;
+				if(!pagewalk(at))
+				{
+					break;
+				}
+
+				std::uint32_t word;
+				if(!avl.dump(at, word))
+				{
+					break;
+				}
+
 				word = (word & 0xff) << 24
 					 | ((word >> 8) & 0xff) << 16
 					 | ((word >> 16) & 0xff) << 8