Merge remote-tracking branch 'upstream/master' into tput

Author: valassi

epoch1/cuda/ee_mumu/SubProcesses/Makefile

@@ -4,60 +4,39 @@ INCFLAGS = -I. -I../../src -I$(TOOLSDIR)
 MODELLIB = model_sm
 OPTFLAGS = -O3 # this ends up in CUFLAGS too (should it?), cannot add -Ofast or -ffast-math here
 OMPFLAGS?= -fopenmp
-CXXFLAGS = $(OPTFLAGS) -std=c++17 $(INCFLAGS) $(USE_NVTX) -Wall -Wshadow -Wextra $(OMPFLAGS) $(MGONGPU_CONFIG)
+CXXFLAGS = $(OPTFLAGS) -std=c++17 $(INCFLAGS) $(USE_NVTX) -Wall -Wshadow -Wextra $(OMPFLAGS)
 CXXFLAGS+= -ffast-math # see issue #117
 ###CXXFLAGS+= -Ofast # performance is not different from --fast-math
 ###CXXFLAGS+= -g # FOR DEBUGGING ONLY
 LIBFLAGS = -L$(LIBDIR) -l$(MODELLIB)
 CXX     ?= g++
 
-# AVX choice (example: "make AVX=none")
-ifneq ($(AVX),)
-###$(info Using AVX='$(AVX)' according to user input)
-else ifeq ($(shell grep -m1 -c avx512vl /proc/cpuinfo)$(shell $(CXX) --version | grep ^clang),1)
-override AVX = 512y
-###$(info Using AVX='$(AVX)' as no user input exists)
-else
-override AVX = avx2
-ifneq ($(shell grep -m1 -c avx512vl /proc/cpuinfo),1)
-$(warning Using AVX='$(AVX)' because host does not support avx512vl)
+# Set the default AVX (vectorization) choice
+ifeq ($(AVX),)
+  ifeq ($(shell grep -m1 -c avx512vl /proc/cpuinfo)$(shell $(CXX) --version | grep ^clang),1)
+    override AVX = 512y
+    ###$(info Using AVX='$(AVX)' as no user input exists)
+  else
+    override AVX = avx2
+    ifneq ($(shell grep -m1 -c avx512vl /proc/cpuinfo),1)
+      $(warning Using AVX='$(AVX)' because host does not support avx512vl)
+    else
+      $(warning Using AVX='$(AVX)' because this is faster than avx512vl for clang)
+    endif
+  endif
 else
-$(warning Using AVX='$(AVX)' because this is faster than avx512vl for clang)
-endif
+  ###$(info Using AVX='$(AVX)' according to user input)
 endif
-###$(info AVX=$(AVX))
 
-# Set the build flags appropriate to each AVX
-# [NB MGONGPU_PVW512 is needed because "-mprefer-vector-width=256" is not exposed in a macro]
-# [See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96476]
-ifeq ($(AVX),sse4)
-override AVXFLAGS = -march=nehalem # SSE4.2 with 128 width (xmm registers)
-else ifeq ($(AVX),avx2)
-override AVXFLAGS = -march=haswell # AVX2 with 256 width (ymm registers) [DEFAULT for clang]
-else ifeq ($(AVX),512y)
-override AVXFLAGS = -march=skylake-avx512 -mprefer-vector-width=256 # AVX512 with 256 width (ymm registers) [DEFAULT for gcc]
-else ifeq ($(AVX),512z)
-override AVXFLAGS = -march=skylake-avx512 -DMGONGPU_PVW512 # AVX512 with 512 width (zmm registers)
-else ifneq ($(AVX),none)
-$(error Unknown AVX='$(AVX)': only 'none', 'sse4', 'avx2', '512y' and '512z' are supported)
+# Set the default FPTYPE (floating point type) choice
+ifeq ($(FPTYPE),)
+  override FPTYPE = d
 endif
 
-# For the moment, use AVXFLAGS everywhere: eventually, use them only in encapsulated implementations
-CXXFLAGS+= $(AVXFLAGS)
-
-# Build tag (defines target and path to the optional build directory)
-override TAG = $(AVX)
-
-# Build directory: current directory by default, or build.$(TAG) if USEBUILDDIR==1
-ifeq ($(USEBUILDDIR),1)
-override BUILDDIR = build.$(TAG)
-override LIBDIR   = ../../lib/$(BUILDDIR)
-else
-override BUILDDIR = .
-override LIBDIR   = ../../lib
+# Set the default RNDGEN (random number generator) choice
+ifeq ($(RNDGEN),)
+  override RNDGEN = curdev
 endif
-###$(info BUILDDIR=$(BUILDDIR))
-$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG))
 
 # If CUDA_HOME is not set, try to set it from the location of nvcc
 ifndef CUDA_HOME
@@ -68,7 +47,6 @@ ifndef CUDA_HOME
     CUDA_HOME := $(warning No CUDA_HOME exported. Using "$(CUDA_HOME)") $(CUDA_HOME)
   endif
 endif
-
 ifneq ($(wildcard $(CUDA_HOME)/bin/nvcc),)
   NVCC = $(CUDA_HOME)/bin/nvcc
   CUARCHNUM=70
@@ -79,7 +57,7 @@ ifneq ($(wildcard $(CUDA_HOME)/bin/nvcc),)
   CUINC       = -I$(CUDA_HOME)/include/
   CULIBFLAGS  = -L$(CUDA_HOME)/lib64/ -lcuda -lcurand
   CUOPTFLAGS  = -lineinfo
-  CUFLAGS     = $(OPTFLAGS) $(CUOPTFLAGS) -std=c++14 $(INCFLAGS) $(CUINC) $(USE_NVTX) $(CUARCHFLAGS) -use_fast_math $(MGONGPU_CONFIG)
+  CUFLAGS     = $(OPTFLAGS) $(CUOPTFLAGS) -std=c++14 $(INCFLAGS) $(CUINC) $(USE_NVTX) $(CUARCHFLAGS) -use_fast_math
   # Without -maxrregcount: baseline throughput: 6.5E8 (16384 32 12) up to 7.3E8 (65536 128 12)
   ###CUFLAGS+= --maxrregcount 160 # improves throughput: 6.9E8 (16384 32 12) up to 7.7E8 (65536 128 12)
   ###CUFLAGS+= --maxrregcount 128 # improves throughput: 7.3E8 (16384 32 12) up to 7.6E8 (65536 128 12)
@@ -92,22 +70,89 @@ else
   NVCC       := $(warning CUDA_HOME is not set or is invalid. Export CUDA_HOME to compile with cuda)
   USE_NVTX   :=
   CULIBFLAGS :=
-  ifndef MGONGPU_CONFIG
-    export MGONGPU_CONFIG = -DMGONGPU_COMMONRAND_ONHOST
-  endif
+  override RNDGEN = common
+endif
+
+# Set the build flags appropriate to each AVX choice (example: "make AVX=none")
+# [NB MGONGPU_PVW512 is needed because "-mprefer-vector-width=256" is not exposed in a macro]
+# [See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96476]
+$(info AVX=$(AVX))
+ifeq ($(AVX),sse4)
+  override AVXFLAGS = -march=nehalem # SSE4.2 with 128 width (xmm registers)
+else ifeq ($(AVX),avx2)
+  override AVXFLAGS = -march=haswell # AVX2 with 256 width (ymm registers) [DEFAULT for clang]
+else ifeq ($(AVX),512y)
+  override AVXFLAGS = -march=skylake-avx512 -mprefer-vector-width=256 # AVX512 with 256 width (ymm registers) [DEFAULT for gcc]
+else ifeq ($(AVX),512z)
+  override AVXFLAGS = -march=skylake-avx512 -DMGONGPU_PVW512 # AVX512 with 512 width (zmm registers)
+else ifneq ($(AVX),none)
+  $(error Unknown AVX='$(AVX)': only 'none', 'sse4', 'avx2', '512y' and '512z' are supported)
+endif
+# For the moment, use AVXFLAGS everywhere: eventually, use them only in encapsulated implementations?
+CXXFLAGS+= $(AVXFLAGS)
+
+# Set the build flags appropriate to each FPTYPE choice (example: "make FPTYPE=f")
+$(info FPTYPE=$(FPTYPE))
+ifeq ($(FPTYPE),d)
+  CXXFLAGS += -DMGONGPU_FPTYPE_DOUBLE
+  CUFLAGS  += -DMGONGPU_FPTYPE_DOUBLE
+else ifeq ($(FPTYPE),f)
+  CXXFLAGS += -DMGONGPU_FPTYPE_FLOAT
+  CUFLAGS  += -DMGONGPU_FPTYPE_FLOAT
+else
+  $(error Unknown FPTYPE='$(FPTYPE)': only 'f' and 'd' are supported)
+endif
+
+# Set the build flags appropriate to each RNDGEN choice (example: "make RNDGEN=common")
+$(info RNDGEN=$(RNDGEN))
+ifeq ($(RNDGEN),curdev)
+  CXXFLAGS += -DMGONGPU_CURAND_ONDEVICE
+  CUFLAGS  += -DMGONGPU_CURAND_ONDEVICE
+else ifeq ($(RNDGEN),curhst)
+  CXXFLAGS += -DMGONGPU_CURAND_ONHOST
+  CUFLAGS  += -DMGONGPU_CURAND_ONHOST
+else ifeq ($(RNDGEN),common)
+  CXXFLAGS += -DMGONGPU_COMMONRAND_ONHOST
+  CUFLAGS  += -DMGONGPU_COMMONRAND_ONHOST
+else
+  $(error Unknown RNDGEN='$(RNDGEN)': only 'curdev', 'curhst' and 'common' are supported)
 endif
 
+# Export AVX, FPTYPE, RNDGEN so that it is not necessary to pass them to the src Makefile too
+export AVX
+export FPTYPE
+export RNDGEN
+
+# Build directory "short" tag (defines target and path to the optional build directory)
+# (Rationale: keep directory names shorter, e.g. do not include random number generator choice)
+override DIRTAG = $(AVX)_$(FPTYPE)
+
+# Build lockfile "full" tag (defines full specification of build options that cannot be intermixed)
+# (Rationale: avoid mixing of CUDA and no-CUDA environment builds with different random number generators)
+override TAG = $(AVX)_$(FPTYPE)_$(RNDGEN)
+
+# Build directory: current directory by default, or build.$(DIRTAG) if USEBUILDDIR==1
+ifeq ($(USEBUILDDIR),1)
+  override BUILDDIR = build.$(DIRTAG)
+  override LIBDIR   = ../../lib/$(BUILDDIR)
+else
+  override BUILDDIR = .
+  override LIBDIR   = ../../lib
+endif
+###$(info BUILDDIR=$(BUILDDIR))
+$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG))
+
 # Enable ccache if USECCACHE=1
 ifeq ($(USECCACHE)$(shell echo $(CXX) | grep ccache),1)
-override CXX:=ccache $(CXX)
+  override CXX:=ccache $(CXX)
 endif
 ifeq ($(USECCACHE)$(shell echo $(AR) | grep ccache),1)
-override AR:=ccache $(AR)
+  override AR:=ccache $(AR)
 endif
 ifneq ($(NVCC),)
-ifeq ($(USECCACHE)$(shell echo $(NVCC) | grep ccache),1)
-override NVCC:=ccache $(NVCC)
-endif
+  ifeq ($(USECCACHE)$(shell echo $(NVCC) | grep ccache),1)
+    override NVCC:=ccache $(NVCC)
+  endif
 endif
 
 GTESTLIBDIR = $(TESTDIR)/googletest/build/lib/
@@ -126,24 +171,24 @@ ifeq ($(UNAME_P),ppc64le)
     CUFLAGS+= -Xcompiler -mno-float128
 endif
 
-all.$(TAG): ../../src/$(BUILDDIR)/.build.tag_$(TAG) $(BUILDDIR)/.build.tag_$(TAG) $(cu_main) $(cxx_main) $(testmain)
+all.$(TAG): ../../src/$(BUILDDIR)/.build.$(TAG) $(BUILDDIR)/.build.$(TAG) $(cu_main) $(cxx_main) $(testmain)
 
-override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.tag_*' ! -name '.build.tag_$(TAG)'`
-$(BUILDDIR)/.build.tag_$(TAG):
+override oldtags=`find $(BUILDDIR) -maxdepth 1 -name '.build.*' ! -name '.build.$(TAG)'`
+$(BUILDDIR)/.build.$(TAG):
 	@if [ ! -d $(BUILDDIR) ]; then echo "mkdir $(BUILDDIR)"; mkdir $(BUILDDIR); fi
-	@if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first or consider using 'make USEBUILDDIR=1 AVX=$(AVX)'"; exit 1; fi
-	@touch $(BUILDDIR)/.build.tag_$(TAG)
+	@if [ "$(oldtags)" != "" ]; then echo -e "Cannot build for tag=$(TAG) as old builds exist for other tags:\n$(oldtags)\nPlease run 'make clean' first\nIf 'make clean' is not enough: run 'make clean USEBUILDDIR=1 AVX=$(AVX) FPTYPE=$(FPTYPE)' or 'make cleanall'"; exit 1; fi
+	@touch $(BUILDDIR)/.build.$(TAG)
 
 debug: OPTFLAGS   = -g -O0 -DDEBUG2
 debug: CUOPTFLAGS = -G
 debug: MAKEDEBUG := debug
 debug: all.$(TAG)
 
-../../src/$(BUILDDIR)/.build.tag_$(TAG):
-	$(MAKE) -C ../../src AVX=$(AVX) $(MAKEDEBUG)
+../../src/$(BUILDDIR)/.build.$(TAG):
+	$(MAKE) -C ../../src $(MAKEDEBUG)
 
 $(LIBDIR)/lib$(MODELLIB).a: ../../src/*.h ../../src/*.cc
-	$(MAKE) -C ../../src AVX=$(AVX) $(MAKEDEBUG)
+	$(MAKE) -C ../../src $(MAKEDEBUG)
 
 $(BUILDDIR)/gcheck.o: gcheck.cu *.h ../../src/*.h ../../src/*.cu
 	@if [ ! -d $(BUILDDIR) ]; then mkdir $(BUILDDIR); fi
@@ -208,17 +253,6 @@ $(GTESTLIBS):
 check: $(testmain)
 	$(testmain)
 
-.PHONY: clean
-
-clean:
-	make -C ../../src AVX=$(AVX) clean
-	rm -f $(BUILDDIR)/.build.tag*
-ifneq ($(BUILDDIR),.)
-	rm -rf $(BUILDDIR)
-else
-	rm -f $(BUILDDIR)/*.o $(BUILDDIR)/*.exe
-endif
-
 avxall:
 	@echo
 	make USEBUILDDIR=1 AVX=none
@@ -231,19 +265,22 @@ avxall:
 	@echo
 	make USEBUILDDIR=1 AVX=512z
 
+.PHONY: clean
+
+clean:
+	make -C ../../src clean
+ifneq ($(BUILDDIR),.)
+	rm -rf $(BUILDDIR)
+else
+	rm -f $(BUILDDIR)/.build.* $(BUILDDIR)/*.o $(BUILDDIR)/*.exe
+endif
+
 cleanall:
 	@echo
 	make clean
 	@echo
-	make USEBUILDDIR=1 AVX=none clean; make -C ../../src USEBUILDDIR=1 AVX=none clean
-	@echo
-	make USEBUILDDIR=1 AVX=sse4 clean; make -C ../../src USEBUILDDIR=1 AVX=sse4 clean
-	@echo
-	make USEBUILDDIR=1 AVX=avx2 clean; make -C ../../src USEBUILDDIR=1 AVX=avx2 clean
-	@echo
-	make USEBUILDDIR=1 AVX=512y clean; make -C ../../src USEBUILDDIR=1 AVX=512y clean
-	@echo
-	make USEBUILDDIR=1 AVX=512z clean; make -C ../../src USEBUILDDIR=1 AVX=512z clean
+	make -C ../../src cleanall
+	rm -rf build.*	
 
 distclean: cleanall
 	make -C $(TOOLSDIR) clean