mk: initial commit

This is a complete overhaul of the build system. This new
implementations upports many new features and is very extensible.

9 files changed, 558 insertions, 0 deletions

diff --git a/mk/build.mk b/mk/build.mk
new file mode 100644
index 0000000..40b2b9e
--- /dev/null
+++ b/mk/build.mk
@@ -0,0 +1,91 @@
+O := build
+src := $(abspath .)
+
+obj = $(O)/$(rule_top)/$(rule_target)-$(build_id)
+build_id = $(call per_target,build_id)
+
+build_vars = $(foreach var,$(1),$(call add_build_var,$(var))$(newline))
+add_build_var = \
+  $(call target_var,build_id_text) += $$(let val,$$($(1)),$$(if $$(val),$(1)="$$(strip $$(val))"))
+
+build_makefiles := $(wildcard mk/*.mk)
+$(build_makefiles):
+
+build_makefiles += Makefile
+build_stack :=
+
+define enter_build
+  build_stack += $$(rule_target);$$(rule_top)
+
+  rule_top := $(1)
+  rule_target := $(if $(2),$(2),$(target))
+  $$(call target_var,build_id_text) =
+endef
+
+define exit_build
+  last_build := $$(lastword $$(build_stack))
+  build_stack := $$(filter-out $$(last_build),$$(build_stack))
+  last_build := $$(subst ;,$(space),$$(last_build))
+
+  rule_top := $$(lastword $$(last_build))
+  rule_target := $$(firstword $$(last_build))
+endef
+
+define setup_obj
+  export build_id_text := $$(strip $$(call per_target,build_id_text))
+  $$(call target_var,build_id) := $$(shell echo -n "$$$$build_id_text" | sha1sum | head -c8)
+  unexport build_id_text
+
+  $$(obj): export CONTENTS := $$(build_id_text)
+  $$(obj):
+	@mkdir -p $$@ && echo -n "$$$$CONTENTS" >$$@/build-vars
+endef
+
+define find_command_lazy
+  $(2)_cmdline := $$($(2))
+  override $(call defer,$(2),$$(call find_command,$(1),$(2)))
+endef
+
+define find_command
+  override $(2) := $$($(2)_cmdline)
+  ifeq (,$$($(2)))
+    override $(2) := $(1)
+  endif
+
+  which_out := $$(shell which $$($(2)) 2>/dev/null)
+
+  ifneq (0,$$(.SHELLSTATUS))
+    which_out :=
+  endif
+
+  ifeq (,$$(which_out))
+    $$(error $(1) ($$($2)) not found)
+  endif
+endef
+
+shell_defer = $(call defer,$(1),$(1) := $$(call shell_checked,$(2)))
+shell_checked = $(shell $(1))$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Command failed: $(1)))
+
+define find_with_pkgconfig
+  pkgs := $(strip $(1))
+
+  ifneq (,$$(pkgs))
+    ifeq (undefined,$$(origin pkgconfig_cflags/$$(pkgs)))
+      $$(eval $$(run_pkgconfig))
+    endif
+
+    $(2) += $$(pkgconfig_cflags/$$(pkgs))
+    $(3) += $$(pkgconfig_libs/$$(pkgs))
+  endif
+endef
+
+define run_pkgconfig
+  pkgconfig_cflags/$$(pkgs) := $$(shell $$(PKG_CONFIG) --cflags $$(pkgs))
+  ifeq (0,$$(.SHELLSTATUS))
+    pkgconfig_libs/$$(pkgs) := $$(shell $$(PKG_CONFIG) --libs $$(pkgs))
+  endif
+
+  ifneq (0,$$(.SHELLSTATUS))
+    $$(error pkg-config failed for package list: $$(pkgs))
+  endif
+endef
diff --git a/mk/cocotb.mk b/mk/cocotb.mk
new file mode 100644
index 0000000..ca95389
--- /dev/null
+++ b/mk/cocotb.mk
@@ -0,0 +1,44 @@
+targets += test
+
+target/test/prepare = $(prepare_verilator_target)
+
+cocotb_modules = $(call per_target,cocotb_modules)
+
+define target/test/setup
+  $(setup_verilator_target)
+
+  $$(call target_var,cocotb_modules) := $$(strip $$(core_info/$$(rule_top)/cocotb_modules))
+
+  ifeq (,$$(cocotb_modules))
+    $$(error core '$$(rule_top)' has no cocotb test modules)
+  endif
+
+  $$(call target_var,vl_main) = $$(cocotb_share)/lib/verilator/verilator.cpp
+  $$(call target_var,vl_flags) += --vpi --public-flat-rw
+  $$(call target_var,vl_ldflags) += \
+    -Wl,-rpath,$$(cocotb_libdir),-rpath,$$(dir $$(cocotb_libpython)) -L$$(cocotb_libdir) \
+    -lcocotbvpi_verilator -lgpi -lcocotb -lgpilog -lcocotbutils $$(cocotb_libpython)
+endef
+
+define target/test/rules
+  $(verilator_target_rules)
+
+  cocotb_outs := $$(addprefix $$(obj)/,results.xml log.txt)
+
+  .PHONY: $$(rule_top_path)/test
+  $$(rule_top_path)/test: $$(obj)/results.xml
+
+  $$(cocotb_outs) &: $$(vtop_exe) | $$(obj)
+	$$(call run_no_err,COCOTB) cd $$(obj) && \
+		LIBPYTHON_LOC=$$(cocotb_libpython) COCOTB_RESULTS_FILE=results.xml \
+		PYTHONPATH="$$(subst $$(space),:,$$(strip $$(cocotb_pythonpath) $$$$PYTHONPATH))" \
+		MODULE=$$(subst $$(space),$$(comma),$$(cocotb_modules)) \
+		$$(src)/$$< >log.txt
+
+  $(call target_entrypoint,$$(cocotb_outs))
+endef
+
+cocotb_pythonpath = \
+  $(addprefix $(src)/, \
+    $(foreach dep,$(dep_tree/$(rule_top)), \
+      $(call core_paths,$(dep),cocotb_paths)))
diff --git a/mk/cores.mk b/mk/cores.mk
new file mode 100644
index 0000000..7ce2e47
--- /dev/null
+++ b/mk/cores.mk
@@ -0,0 +1,112 @@
+here = $(if $(mod_path),$(mod_path)/)
+mod_path :=
+subdir_stack :=
+
+unknown_core = $(error unknown core '$(1)')
+
+all_cores :=
+all_stamps :=
+
+top_stamp = $(call core_stamp,$(rule_top))
+core_stamp = $(obj)/deps/$(core_info/$(1)/path)/stamp
+
+core_paths = \
+  $(patsubst /%,%, \
+    $(patsubst /,., \
+      $(abspath \
+        $(let prefix,$(core_info/$(1)/workdir), \
+          $(addprefix /$(if $(prefix),$(prefix)/),$(core_info/$(1)/$(2)))))))
+
+define add_core
+  this := core_info/$(1)
+
+  ifneq (,$$($$(this)/path))
+    $$(error multiple definitions of core '$(1)': '$$($$(this)/path)' and '$(2)')
+  else ifneq (,$$(core_path/$(2)))
+    $$(error multiple cores under path '$(2)')
+  endif
+
+  $$(this)/path := $(2)
+  $$(this)/mod_file := $$(mod_file)
+  $$(this)/workdir := $$(mod_path)
+
+  $$(eval $$(call $(3)))
+
+  this :=
+  all_cores += $(1)
+  core_path/$(2) := $(1)
+endef
+
+define add_core_subdir
+  core :=
+  cores :=
+  subdirs :=
+
+  subdir_stack += $$(mod_path)
+  mod_path := $$(here)$(1)
+  mod_file := $$(here)mod.mk
+
+  include $$(mod_file)
+
+  $$(if $$(core), \
+    $$(eval $$(call add_core,$(notdir $(1)),$$(mod_path),core)))
+
+  $$(foreach core,$$(cores), \
+    $$(eval $$(call add_core,$$(core),$$(here)$$(core),core/$$(core))))
+
+  $$(foreach subdir,$$(subdirs), \
+    $$(eval $$(call add_core_subdir,$$(subdir))))
+
+  mod_path := $$(lastword $$(subdir_stack))
+  subdir_stack := $$(filter-out $$(mod_path),$$(subdir_stack))
+endef
+
+define setup_dep_tree
+  $$(foreach core,$$(all_cores), \
+    $$(eval $$(call defer,dep_tree/$$(core),$$$$(call get_core_deps,$$(core)))))
+endef
+
+define setup_stamp_rules
+  $$(foreach core,$$(all_cores), \
+    $$(let stamp,$$(call core_stamp,$$(core)), \
+      $$(stamp) \
+      $$(eval $$(call add_core_stamp,$$(core),$$(stamp))))): $$(build_makefiles) | $$(obj)
+endef
+
+define add_core_stamp
+  $(2): $$(core_info/$(1)/mod_file) \
+        $$(foreach dep,$$(core_info/$(1)/deps),$$(call core_stamp,$$(dep)))
+
+  all_stamps += $(2)
+endef
+
+define get_core_deps
+  dep_tree/$(1) :=
+
+  $$(foreach dep,$$(core_info/$(1)/deps), \
+    $$(if $$(core_info/$$(dep)/path),,$$(call unknown_core,$$(dep))) \
+    $$(eval dep_tree/$(1) := \
+      $$(dep_tree/$$(dep)) $$(filter-out $$(dep_tree/$$(dep)),$$(dep_tree/$(1)))))
+
+  dep_tree/$(1) := $$(strip $$(dep_tree/$(1)))
+  dep_tree/$(1) += $(1)
+endef
+
+map_core_deps = \
+  $(if $(findstring undefined,$(origin $(1)_deps/$(2))), \
+    $(eval $(call merge_mapped_deps,$(1),$(2)))) \
+  $($(1)_deps/$(2))
+
+define merge_mapped_deps
+  $(1)_deps/$(2) := $$(core_info/$(2)/$(1))
+
+  $$(foreach dep,$$(core_info/$(2)/deps), \
+    $$(eval $(1)_deps/$(2) := \
+      $$(let mapped_dep,$$(call map_core_deps,$(1),$$(dep)), \
+        $$(mapped_dep) $$(filter-out $$(mapped_dep),$$($(1)_deps/$(2))))))
+endef
+
+define finish_stamp_rules
+  $$(all_stamps):
+	@mkdir -p $$$$(dirname $$@) && touch $$@
+endef
diff --git a/mk/cov.mk b/mk/cov.mk
new file mode 100644
index 0000000..1281e4e
--- /dev/null
+++ b/mk/cov.mk
@@ -0,0 +1,30 @@
+targets += cov
+
+cov_cores = $(call per_target,cov_cores)
+
+define target/cov/prepare
+  enable_cov := 1
+endef
+
+define target/cov/setup
+  $$(call target_var,cov_cores) := \
+    $$(foreach dep,$$(dep_tree/$$(rule_top)), \
+      $$(if $$(filter test,$$(core_info/$$(dep)/targets)), \
+        $$(eval $$(call build_target_top,$$(dep),test)) \
+        $$(dep)))
+endef
+
+define target/cov/rules
+  .PHONY: $$(rule_top_path)/cov
+  $$(rule_top_path)/cov: $$(obj)/html
+
+  $$(obj)/html: $$(obj)/coverage.info | $$(obj)
+	@rm -rf $$@
+	$$(call run,GENHTML) $$(GENHTML) $$< --output-dir=$$@
+
+  $$(obj)/coverage.info: $$(foreach core,$$(cov_cores),$$(obj/test/$$(core))/results.xml) | $$(obj)
+	$$(call run,COVERAGE) $$(VERILATOR)_coverage -write-info $$@ \
+		$$(wildcard $$(foreach core,$$(cov_cores),$$(obj/test/$$(core))/coverage.dat))
+
+  $(call target_entrypoint,$(obj)/html)
+endef
diff --git a/mk/output.mk b/mk/output.mk
new file mode 100644
index 0000000..d2a3f2f
--- /dev/null
+++ b/mk/output.mk
@@ -0,0 +1,12 @@
+$(V).SILENT:
+
+run = \
+  $(call run_common,$(1),$(2),$(3)) \
+  $(if $(V),$(newline)$(3),; trap 'echo "Exited with code $$?: $$BASH_COMMAND" >&2' ERR;)
+
+run_no_err = $(call run_common,$(1),$(2),$(3))$(newline)$(3)
+
+run_common = \
+  $(3)@printf '%s %-7s %-9s %s\n' '$(build_id)' '($(rule_target))' '$(1)' '$(if $(2),$(2),$(rule_top_path))'
+
+run_submake = $(call run_no_err,$(1),$(2),+)$(MAKE)
diff --git a/mk/target.mk b/mk/target.mk
new file mode 100644
index 0000000..d28c2e2
--- /dev/null
+++ b/mk/target.mk
@@ -0,0 +1,45 @@
+target_var = $(1)/$(rule_target)/$(rule_top)
+per_target = $($(call target_var,$(1)))
+
+rule_top_path = $(core_info/$(rule_top)/path)
+
+define target_entrypoint
+  $(1): rule_top := $$(rule_top)
+  $(1): rule_target := $$(rule_target)
+endef
+
+define check_target
+  ifneq ($$(target),$$(findstring $$(target),$$(targets)))
+    $$(error bad target '$$(target)')
+  endif
+endef
+
+define setup_submake_rules
+  .PHONY: $$(targets)
+
+  other_targets := $$(filter-out $$(target),$$(targets))
+
+  $$(foreach t,$$(targets),$$(eval $$(call top_rule,$$(t))))
+
+  ifeq (,$$(target))
+    $$(foreach other,$$(other_targets), \
+      $$(foreach core,$$(all_cores), \
+        $$(eval $$(call submake_rule,$$(other),$$(core)))))
+  else
+    $$(foreach core,$$(filter-out $$(top),$$(all_cores)), \
+      $$(eval $$(call submake_rule,$$(target),$$(core))))
+  endif
+endef
+
+define top_rule
+  $(1): $$(top_path)/$(1)
+endef
+
+define submake_rule
+  path := $$(core_info/$(2)/path)/$(1)
+
+  .PHONY: $$(path)
+
+  $$(path):
+	+$$(MAKE) --no-print-directory target=$(1) top=$(2) $$@
+endef
diff --git a/mk/tools.mk b/mk/tools.mk
new file mode 100644
index 0000000..9f3734e
--- /dev/null
+++ b/mk/tools.mk
@@ -0,0 +1,10 @@
+define find_tools_lazy
+  $(call find_command_lazy,cocotb-config,COCOTB_CONFIG)
+  $(call find_command_lazy,genhtml,GENHTML)
+  $(call find_command_lazy,pkg-config,PKG_CONFIG)
+  $(call find_command_lazy,verilator,VERILATOR)
+
+  $(call shell_defer,cocotb_share,$$(COCOTB_CONFIG) --share)
+  $(call shell_defer,cocotb_libdir,$$(COCOTB_CONFIG) --lib-dir)
+  $(call shell_defer,cocotb_libpython,$$(COCOTB_CONFIG) --libpython)
+endef
diff --git a/mk/top.mk b/mk/top.mk
new file mode 100644
index 0000000..6db6abb
--- /dev/null
+++ b/mk/top.mk
@@ -0,0 +1,78 @@
+.PHONY: .force .no_default
+
+.no_target:
+	$(error no default target defined in top Makefile)
+
+.force:
+
+empty :=
+space := $(empty) $(empty)
+comma := ,
+
+# Both empty lines are required
+define newline
+
+
+endef
+
+newline := $(newline)
+
+defer = $(1) = $$(eval $(2))$$($(1))
+
+ifeq (,$(top))
+  $(error $$(top) is not defined)
+endif
+
+$(foreach flag,$(subst $(comma),$(space),$(enable)),$(eval override enable_$(flag) := 1))
+$(foreach flag,$(subst $(comma),$(space),$(disable)),$(eval override enable_$(flag) :=))
+
+include mk/build.mk
+include mk/cocotb.mk
+include mk/cores.mk
+include mk/cov.mk
+include mk/output.mk
+include mk/target.mk
+include mk/tools.mk
+include mk/verilator.mk
+
+$(eval $(check_target))
+$(eval $(find_tools_lazy))
+
+ifneq (,$(target))
+  $(eval $(target/$(target)/prepare))
+endif
+
+$(foreach top_dir,$(core_dirs), \
+  $(eval $(call add_core_subdir,$(top_dir))))
+
+top_path := $(core_info/$(top)/path)
+
+ifeq (,$(top_path))
+  $(call unknown_core,$(top))
+endif
+
+$(eval $(setup_dep_tree))
+
+define build_target_top
+  ifeq (,$$(obj/$(if $(2),$(2),$(target))/$(1)))
+    $$(eval $$(call enter_build,$(1),$(2)))
+    $$(eval $$(call build_vars,rule_target rule_top core_info/$$(rule_top)/build))
+
+    $$(eval $$(target/$$(rule_target)/setup))
+
+    $$(eval $$(setup_obj))
+    $$(eval $$(setup_stamp_rules))
+
+    $$(eval $$(target/$$(rule_target)/rules))
+
+    obj/$$(rule_target)/$$(rule_top) := $$(obj)
+    $$(eval $$(exit_build))
+  endif
+endef
+
+ifneq (,$(target))
+  $(eval $(call build_target_top,$(top)))
+endif
+
+$(eval $(setup_submake_rules))
+$(eval $(finish_stamp_rules))
diff --git a/mk/verilator.mk b/mk/verilator.mk
new file mode 100644
index 0000000..0205e74
--- /dev/null
+++ b/mk/verilator.mk
@@ -0,0 +1,136 @@
+targets += sim
+
+vtop_dir = $(call per_target,vtop_dir)
+vtop_exe = $(call per_target,vtop_exe)
+
+vl_main = $(call per_target,vl_main)
+vl_flags = $(call per_target,vl_flags)
+vl_cflags = $(call per_target,vl_cflags)
+vl_ldflags = $(call per_target,vl_ldflags)
+
+define target/sim/prepare
+  enable_opt := 1
+
+  $(prepare_verilator_target)
+endef
+
+define target/sim/setup
+  $(setup_verilator_target)
+
+  $$(call target_var,vl_main) := $$(strip $$(call core_paths,$$(rule_top),vl_main))
+  ifeq (,$$(vl_main))
+    $$(error core '$$(rule_top)' does not define vl_main)
+  endif
+endef
+
+define target/sim/rules
+  $(verilator_target_rules)
+
+  .PHONY: $$(rule_top_path)/sim
+
+  $$(rule_top_path)/sim: $$(vtop_exe)
+	$$<
+endef
+
+define prepare_verilator_target
+  flow/type := sim
+endef
+
+define setup_verilator_target
+  $(call build_vars,$(addprefix enable_,rand threads trace cov opt lto))
+
+  $(call target_var,vl_flags) = $(common_vl_flags)
+  $(call target_var,vl_cflags) = $(common_vl_cflags)
+  $(call target_var,vl_ldflags) = $(common_vl_ldflags)
+endef
+
+$(eval $(call defer,common_vl_flags,$$(set_verilator_common)))
+$(eval $(call defer,common_vl_cflags,$$(set_verilator_common)))
+$(eval $(call defer,common_vl_ldflags,$$(set_verilator_common)))
+
+define set_verilator_common
+  ifneq (,$$(enable_lto))
+    enable_opt := 1
+  endif
+
+  x_mode := $$(if $$(enable_rand),unique,fast)
+
+  static_flags := \
+    --x-assign $$(x_mode) --x-initial $$(x_mode) \
+    $$(if $$(enable_threads),--threads $$(call shell_checked,nproc)) \
+    $$(if $$(enable_trace),--trace --trace-fst --trace-structs) \
+    $$(if $$(enable_cov),--coverage) \
+    $$(if $$(enable_opt),-O3) \
+    --cc --exe --prefix Vtop --MMD --MP
+
+  common_vl_flags := $$(static_flags) $$(core_info/$$(rule_top)/vl_flags)
+
+  common_vl_cflags := \
+    $$(if $$(enable_opt),-O3) \
+    $$(if $$(enable_lto),-flto)
+
+  common_vl_ldflags := \
+    $$(if $$(enable_lto),-flto)
+endef
+
+define verilator_target_rules
+  $(call target_var,vtop_dir) := $$(obj)/vl
+  $(call target_var,vtop_exe) := $$(vtop_dir)/Vtop
+
+  vtop_mk_file := $$(vtop_dir)/Vtop.mk
+  vtop_mk_stamp := $$(vtop_dir)/stamp
+  vtop_dep_file := $$(vtop_dir)/Vtop__ver.d
+
+  -include $$(vtop_dep_file)
+  $$(vtop_dep_file):
+
+  $$(vtop_exe): export VPATH := $$(src)
+  $$(vtop_exe): $$(vtop_mk_stamp)
+	$$(call run_submake,BUILD) $$(if $$(V),,-s) -C $$(vtop_dir) -f Vtop.mk
+	@touch -c $$@
+
+  $$(vtop_mk_file):
+	@rm -f $$@
+
+  $$(vtop_mk_stamp): $$(top_stamp) $$(vtop_mk_file)
+	$$(eval $$(final_vflags))
+	$$(call run,VERILATE) $$(VERILATOR) $$(vl_flags) $$(verilator_src_args)
+	@touch $$@
+
+  $(call target_entrypoint,$$(vtop_exe))
+endef
+
+define final_vflags
+  $(call find_with_pkgconfig, \
+    $(call map_core_deps,vl_pkgconfig,$(rule_top)), \
+    $(call target_var,vl_cflags), \
+    $(call target_var,vl_ldflags))
+
+  $$(call target_var,vl_flags) += --Mdir $$(vtop_dir)
+  $$(call target_var,vl_cflags) := $$(strip $$(vl_cflags))
+  $$(call target_var,vl_ldflags) := $$(strip $$(vl_ldflags))
+
+  # Verilator's wrapper script can't handle `-CFLAGS ''` correctly
+  ifneq (,$$(vl_cflags))
+    $$(call target_var,vl_flags) += -CFLAGS '$$(vl_cflags)'
+  endif
+
+  ifneq (,$$(vl_ldflags))
+    $$(call target_var,vl_flags) += -LDFLAGS '$$(vl_ldflags)'
+  endif
+endef
+
+verilator_src_args = \
+  $(strip \
+    $(let rtl_top,$(core_info/$(rule_top)/rtl_top), \
+      $(if $(rtl_top),--top $(rtl_top),$(error core '$(rule_top)' must define rtl_top)) \
+      $(foreach dep,$(dep_tree/$(rule_top)), \
+        $(let prefix,$(core_info/$(dep)/workdir)/, \
+          $(foreach rtl_dir,$(call core_paths,$(dep),rtl_dirs), \
+            -y $(rtl_dir)) \
+          $(foreach include_dir,$(call core_paths,$(dep),rtl_include_dirs), \
+            -I$(include_dir)) \
+          $(foreach src_file,$(call core_paths,$(dep),rtl_files) $(call core_paths,$(dep),vl_files), \
+            $(src_file)))) \
+      $(if $(core_info/$(rule_top)/rtl_files),,$(rtl_top))) \
+    $(if $(vl_main),$(vl_main),$(error $$(vl_main) not defined by target '$(rule_target)')))