commit 4f271849224b5a5b3ca93c78e4b5ca8aee350bc6 Author: Jonas Eriksson Date: Wed Mar 26 17:29:36 2014 +0100 cpuflags: Avoid building on test target The cpuflags-test utility is built with different compiler flags depending on the callee, meaning that pre-building is tricky. By separating the builds from each other using a suffix based on the extra flags, we can pre-build multiple versions of the application. Signed-off-by: Jonas Eriksson Upstream-Status: Pending diff --git a/provider/cpuflags.py b/provider/cpuflags.py index 70255cf..ea2031c 100644 --- a/provider/cpuflags.py +++ b/provider/cpuflags.py @@ -2,6 +2,7 @@ Shared code for tests that make use of cpuflags """ import os +import md5 from virttest import data_dir @@ -12,19 +13,36 @@ def install_cpuflags_util_on_vm(test, vm, dst_dir, extra_flags=None): :param vm: virtual machine. :param dst_dir: Installation path. :param extra_flags: Extraflags for gcc compiler. + :return: The full path to the cpu_flags command built with the supplied + extra_flags """ if not extra_flags: extra_flags = "" - cpuflags_src = data_dir.get_deps_dir("cpu_flags") + # The suffix is the first 8 chars of the hex representation of the md5sum + # of the extra flags + extra_flags_hash = md5.new() + extra_flags_hash.update(extra_flags) + suffix = (extra_flags_hash.hexdigest())[:8] + cpuflags_dst = os.path.join(dst_dir, "cpu_flags") + cpuflags_cmd = os.path.join(dst_dir, "cpuflags-test-%s" % suffix) session = vm.wait_for_login() - session.cmd("rm -rf %s" % - (cpuflags_dst)) - session.cmd("sync") - vm.copy_files_to(cpuflags_src, dst_dir) - session.cmd("sync") - session.cmd("cd %s; cd src; make EXTRA_FLAGS='%s';" % - (cpuflags_dst, extra_flags)) - session.cmd("sync") + + if session.cmd_status("test -x '%s'" % cpuflags_cmd): + cpuflags_cmd = os.path.join(cpuflags_dst, "cpuflags-test-%s" % suffix) + if session.cmd_status("test -x '%s'" % cpuflags_cmd): + cpuflags_src = os.path.join(data_dir.get_deps_dir(), "cpu_flags") + + session.cmd("rm -rf %s" % (cpuflags_dst)) + session.cmd("mkdir -p %s" % (dst_dir)) + session.cmd("sync") + vm.copy_files_to(cpuflags_src, dst_dir) + session.cmd("sync") + session.cmd("make -C %s EXTRA_FLAGS='%s';" % + (cpuflags_dst, extra_flags)) + session.cmd("sync") + session.close() + + return cpuflags_cmd diff --git a/deps/cpu_flags/src/.gitignore b/deps/cpu_flags/src/.gitignore new file mode 100644 index 0000000..1d04e1f --- /dev/null +++ b/qemu/deps/cpu_flags/src/.gitignore @@ -0,0 +1,2 @@ +build-*/ +cpuflags-test-* diff --git a/deps/cpu_flags/src/Makefile b/deps/cpu_flags/src/Makefile index 1513649..3be1920 100644 --- a/deps/cpu_flags/src/Makefile +++ b/deps/cpu_flags/src/Makefile @@ -45,63 +45,77 @@ CC=gcc LIBS=-lgomp -.PHONY: default all cpuflags-test clean +# Use the first 8 characters of the md5sum of the extra flags as suffix +SUFFIX=$(shell echo -n '$(EXTRA_FLAGS)' | md5sum | cut -c 1-8) +BUILDDIR=build-$(SUFFIX) +BUILDDIR_IND=build-$(SUFFIX)/.created -default:cpuflags-test +.PHONY: default all clean suffix -all:cpuflags-test +default: cpuflags-test-$(SUFFIX) -cpuflags-test: avx.o fma4.o xop.o sse4a.o sse4.o ssse3.o sse3.o aes.o pclmul.o rdrand.o stress.o stressmem.o - $(CC) $(CFLAGS) $(LIBS) cpuflags-test.c -o cpuflags-test \ - aes.o \ - pclmul.o \ - rdrand.o \ - avx.o \ - fma4.o \ - xop.o \ - sse4a.o \ - sse4.o \ - ssse3.o \ - sse3.o \ - stress.o \ - stressmem.o \ +all: cpuflags-test-$(SUFFIX) -aes.o: aes.c tests.h - $(CC) $(CFLAGSAES) $(LIBS) -c aes.c +suffix: + @echo $(SUFFIX) -pclmul.o: pclmul.c tests.h - $(CC) $(CFLAGSPCLMUL) $(LIBS) -c pclmul.c +# The timestamp of builddir will be updated as we write files, so let the +# creation of .o files depend on a file (builddir indicator) inside builddir to +# make sure that builddir is created. +$(BUILDDIR_IND): + $(MKDIR) $(BUILDDIR) + touch $(BUILDDIR_IND) -rdrand.o: rdrand.c tests.h - $(CC) $(CFLAGSRDRAND) $(LIBS) -c rdrand.c +cpuflags-test-$(SUFFIX): cpuflags-test.c \ + $(BUILDDIR)/avx.o \ + $(BUILDDIR)/fma4.o \ + $(BUILDDIR)/xop.o \ + $(BUILDDIR)/sse4a.o \ + $(BUILDDIR)/sse4.o \ + $(BUILDDIR)/ssse3.o \ + $(BUILDDIR)/sse3.o \ + $(BUILDDIR)/aes.o \ + $(BUILDDIR)/pclmul.o \ + $(BUILDDIR)/rdrand.o \ + $(BUILDDIR)/stress.o \ + $(BUILDDIR)/stressmem.o + $(CC) $(CFLAGS) $^ $(LIBS) -o $@ -fma4.o: fma4.c tests.h - $(CC) $(CFLAGSFMA4) $(LIBS) -c fma4.c +$(BUILDDIR)/aes.o: aes.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSAES) $(LIBS) -c aes.c -xop.o: xop.c tests.h - $(CC) $(CFLAGSXOP) $(LIBS) -c xop.c +$(BUILDDIR)/pclmul.o: pclmul.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSPCLMUL) $(LIBS) -c pclmul.c -avx.o: avx.c tests.h - $(CC) $(CFLAGSAVX) $(LIBS) -c avx.c +$(BUILDDIR)/rdrand.o: rdrand.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSRDRAND) $(LIBS) -c rdrand.c -sse4a.o: sse4a.c tests.h - $(CC) $(CFLAGSSSE4A) $(LIBS) -c sse4a.c +$(BUILDDIR)/fma4.o: fma4.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSFMA4) $(LIBS) -c fma4.c -sse4.o: sse4.c tests.h - $(CC) $(CFLAGSSSE4) $(LIBS) -c sse4.c +$(BUILDDIR)/xop.o: xop.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSXOP) $(LIBS) -c xop.c -ssse3.o: ssse3.c tests.h - $(CC) $(CFLAGSSSSE3) $(LIBS) -c ssse3.c +$(BUILDDIR)/avx.o: avx.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSAVX) $(LIBS) -c avx.c -sse3.o: sse3.c tests.h - $(CC) $(CFLAGSSSE3) $(LIBS) -c sse3.c +$(BUILDDIR)/sse4a.o: sse4a.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSSE4A) $(LIBS) -c sse4a.c -stress.o: stress.c tests.h - $(CC) $(CFLAGSSTRESS) $(LIBS) -c stress.c +$(BUILDDIR)/sse4.o: sse4.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSSE4) $(LIBS) -c sse4.c -stressmem.o: stressmem.c tests.h - $(CC) $(CFLAGSSTRESS) $(LIBS) -c stressmem.c +$(BUILDDIR)/ssse3.o: ssse3.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSSSE3) $(LIBS) -c ssse3.c +$(BUILDDIR)/sse3.o: sse3.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSSE3) $(LIBS) -c sse3.c + +$(BUILDDIR)/stress.o: stress.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSTRESS) $(LIBS) -c stress.c + +$(BUILDDIR)/stressmem.o: stressmem.c tests.h $(BUILDDIR_IND) + $(CC) -o $@ $(CFLAGSSTRESS) $(LIBS) -c stressmem.c ARCHIVE= cpuflags-test @@ -110,5 +124,7 @@ tar: clean clean: rm -f *~ - rm -f *.o - rm -f cpuflags-test + rm -f build-*/*.o + rm -f cpuflags-test-* + rm -f build-*/.created + rmdir build-* || true diff --git a/qemu/tests/cpuflags.py b/qemu/tests/cpuflags.py index 75a03bb..cab4f5a 100644 --- a/qemu/tests/cpuflags.py +++ b/qemu/tests/cpuflags.py @@ -24,8 +24,8 @@ def run(test, params, env): utils_misc.Flag.aliases = utils_misc.kvm_map_flags_aliases qemu_binary = utils_misc.get_qemu_binary(params) - cpuflags_src = os.path.join(data_dir.get_deps_dir("cpu_flags"), "src") - cpuflags_def = os.path.join(data_dir.get_deps_dir("cpu_flags"), + cpuflags_src = os.path.join(qemudir, "deps", "cpu_flags", "src") + cpuflags_def = os.path.join(qemudir, "deps", "cpu_flags", "cpu_map.xml") smp = int(params.get("smp", 1)) @@ -188,9 +188,12 @@ def run(test, params, env): """ p = ParseCpuFlags() cpus = p.parse_file(cpuflags_def) + flags = None for arch in cpus.values(): if cpumodel in arch.keys(): flags = arch[cpumodel] + if not flags: + raise NameError, "Could not resolve flags for cpumodel '%s'" % cpumodel return set(map(utils_misc.Flag, flags)) get_guest_host_cpuflags_BAD = get_guest_host_cpuflags_1350 @@ -253,7 +256,9 @@ def run(test, params, env): output = utils.run(cmd).stdout cpu_re = re.compile(r"x86\s+\[?(\w+)\]?") - return cpu_re.findall(output) + cpus = cpu_re.findall(output) + + return [c for c in cpus if not c == 'host'] get_cpu_models_BAD = get_cpu_models_1350 @@ -432,16 +437,16 @@ def run(test, params, env): Run stress on vm for timeout time. """ ret = False - install_path = "/tmp" - cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) - flags = check_cpuflags_work(vm, install_path, guest_flags) + install_path = params.get("cpuflags_install_path", "/tmp") + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) + flags = check_cpuflags_work(vm, cpuflags_cmd, guest_flags) dd_session = vm.wait_for_login() stress_session = vm.wait_for_login() dd_session.sendline("dd if=/dev/[svh]da of=/tmp/stressblock" " bs=10MB count=100 &") try: - stress_session.cmd("%s/cpuflags-test --stress %s%s" % - (os.path.join(install_path, "cpu_flags"), smp, + stress_session.cmd("%s --stress %s%s" % + (cpuflags_cmd, smp, utils_misc.kvm_flags_to_stresstests(flags[0])), timeout=timeout) except aexpect.ShellTimeoutError: @@ -613,10 +618,10 @@ def run(test, params, env): "on guest: %s", str(not_enable_flags)) logging.info("Check main instruction sets.") - install_path = "/tmp" - cpuflags.install_cpuflags_util_on_vm(test, self.vm, install_path) + install_path = params.get("cpuflags_install_path", "/tmp") + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) - Flags = check_cpuflags_work(self.vm, install_path, + Flags = check_cpuflags_work(self.vm, cpuflags_cmd, flags.all_possible_guest_flags) logging.info("Woking CPU flags: %s", str(Flags[0])) logging.info("Not working CPU flags: %s", str(Flags[1])) @@ -814,18 +819,17 @@ def run(test, params, env): (self.vm, _) = start_guest_with_cpuflags(cpuf_model, smp) - install_path = "/tmp" - cpuflags.install_cpuflags_util_on_vm(test, self.vm, install_path) - flags = check_cpuflags_work(self.vm, install_path, + install_path = params.get("cpuflags_install_path", "/tmp") + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, self.vm, install_path) + flags = check_cpuflags_work(self.vm, cpuflags_cmd, flags.guest_flags) dd_session = self.vm.wait_for_login() stress_session = self.vm.wait_for_login() dd_session.sendline("nohup dd if=/dev/[svh]da of=/tmp/" "stressblock bs=10MB count=100 &") - cmd = ("nohup %s/cpuflags-test --stress %s%s &" % - (os.path.join(install_path, "cpu_flags"), smp, - utils_misc.kvm_flags_to_stresstests(flags[0]))) + cmd = "nohup %s --stress %s%s &" % (cpuflags_cmd, smp, + utils_misc.kvm_flags_to_stresstests(flags[0])) stress_session.sendline(cmd) time.sleep(5) @@ -915,7 +919,7 @@ def run(test, params, env): for fdel in flags.host_unsupported_flags: cpuf_model += ",-" + str(fdel) - install_path = "/tmp" + install_path = params.get("cpuflags_install_path", "/tmp") class testMultihostMigration(utils_test.qemu.MultihostMigration): @@ -931,9 +935,10 @@ def run(test, params, env): vm = env.get_vm("vm1") session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) + mig_data.params.update([('cpuflags_cmd', cpuflags_cmd)]) - Flags = check_cpuflags_work(vm, install_path, + Flags = check_cpuflags_work(vm, cpuflags_cmd, flags.all_possible_guest_flags) logging.info("Woking CPU flags: %s", str(Flags[0])) logging.info("Not working CPU flags: %s", @@ -946,9 +951,8 @@ def run(test, params, env): session.sendline("nohup dd if=/dev/[svh]da of=/tmp/" "stressblock bs=10MB count=100 &") - cmd = ("nohup %s/cpuflags-test --stress %s%s &" % - (os.path.join(install_path, "cpu_flags"), - smp, + cmd = ("nohup %s --stress %s%s &" % + (cpuflags_cmd, smp, utils_misc.kvm_flags_to_stresstests(Flags[0] & flags.guest_flags))) logging.debug("Guest_flags: %s", @@ -971,7 +975,8 @@ def run(test, params, env): " should be active after" " migration and it's not.") - Flags = check_cpuflags_work(vm, install_path, + cpuflags_cmd = mig_data.params.get('cpuflags_cmd') + Flags = check_cpuflags_work(vm, cpuflags_cmd, flags.all_possible_guest_flags) logging.info("Woking CPU flags: %s", str(Flags[0])) @@ -1017,7 +1022,7 @@ def run(test, params, env): disable_cpus = map(lambda cpu: int(cpu), params.get("disable_cpus", "").split()) - install_path = "/tmp" + install_path = params.get("cpuflags_install_path", "/tmp") class testMultihostMigration(utils_test.qemu.MultihostMigration): @@ -1060,9 +1065,10 @@ def run(test, params, env): vm = env.get_vm("vm1") session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path) + mig_data.params.update([('cpuflags_cmd', cpuflags_cmd)]) - Flags = check_cpuflags_work(vm, install_path, + Flags = check_cpuflags_work(vm, cpuflags_cmd, flags.all_possible_guest_flags) logging.info("Woking CPU flags: %s", str(Flags[0])) logging.info("Not working CPU flags: %s", @@ -1103,7 +1109,8 @@ def run(test, params, env): "disabled on dsthost:%s" % (disable_cpus, not_disabled)) - Flags = check_cpuflags_work(vm, install_path, + cpuflags_cmd = mig_data.params.get('cpuflags_cmd') + Flags = check_cpuflags_work(vm, cpuflags_cmd, flags.all_possible_guest_flags) logging.info("Woking CPU flags: %s", str(Flags[0])) diff --git a/qemu/tests/migration_multi_host_cancel.py b/qemu/tests/migration_multi_host_cancel.py index 89765dd..a4ed11b 100644 --- a/qemu/tests/migration_multi_host_cancel.py +++ b/qemu/tests/migration_multi_host_cancel.py @@ -64,13 +64,12 @@ def run(test, params, env): vm = mig_data.vms[0] session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, self.install_path, extra_flags="-msse3 -msse2") - cmd = ("%s/cpuflags-test --stressmem %d,%d %%" % - (os.path.join(self.install_path, "cpu_flags"), - self.vm_mem * 10, self.vm_mem / 2)) + cmd = ("%s --stressmem %d,%d %%" % + (cpuflags_cmd, self.vm_mem * 10, self.vm_mem / 2)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd) diff --git a/qemu/tests/migration_multi_host_downtime_and_speed.py b/qemu/tests/migration_multi_host_downtime_and_speed.py index 4a06b69..549d1fd 100644 --- a/qemu/tests/migration_multi_host_downtime_and_speed.py +++ b/qemu/tests/migration_multi_host_downtime_and_speed.py @@ -164,13 +164,12 @@ def run(test, params, env): vm = mig_data.vms[0] session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, self.install_path, extra_flags="-msse3 -msse2") - cmd = ("nohup %s/cpuflags-test --stressmem %d,%d &" % - (os.path.join(self.install_path, "cpu_flags"), - self.vm_mem * 100, self.vm_mem / 2)) + cmd = ("nohup %s --stressmem %d,%d &" % + (cpuflags_cmd, self.vm_mem * 100, self.vm_mem / 2)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd) time.sleep(3) diff --git a/qemu/tests/migration_multi_host_firewall_block.py b/qemu/tests/migration_multi_host_firewall_block.py index 3db1dc5..dc70a6b 100644 --- a/qemu/tests/migration_multi_host_firewall_block.py +++ b/qemu/tests/migration_multi_host_firewall_block.py @@ -175,13 +175,12 @@ def run(test, params, env): vm = mig_data.vms[0] session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, self.install_path, extra_flags="-msse3 -msse2") - cmd = ("nohup %s/cpuflags-test --stressmem %d,%d &" % - (os.path.join(self.install_path, "cpu_flags"), - self.vm_mem * 100, self.vm_mem / 2)) + cmd = ("nohup %s --stressmem %d,%d &" % + (cpuflags_cmd, self.vm_mem * 100, self.vm_mem / 2)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd) time.sleep(3) diff --git a/qemu/tests/migration_multi_host_ping_pong.py b/qemu/tests/migration_multi_host_ping_pong.py index 7f7b64e..c6d0d3c 100644 --- a/qemu/tests/migration_multi_host_ping_pong.py +++ b/qemu/tests/migration_multi_host_ping_pong.py @@ -155,15 +155,14 @@ def run(test, params, env): vm = mig_data.vms[0] session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, self.install_path, extra_flags="-msse3 -msse2") cmd = ("nohup %s/cpuflags-test --stressmem %d,32" " > %s &" % - (os.path.join(self.install_path, "cpu_flags"), - self.stress_memory, - self.cpuflags_test_out)) + (cpuflags_cmd, self.stress_memory, + self.cpuflags_test_out)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd) if session.cmd_status("killall -s 0 cpuflags-test") != 0: diff --git a/qemu/tests/migration_multi_host_with_speed_measurement.py b/qemu/tests/migration_multi_host_with_speed_measurement.py index e845b90..dbddbf4 100644 --- a/qemu/tests/migration_multi_host_with_speed_measurement.py +++ b/qemu/tests/migration_multi_host_with_speed_measurement.py @@ -128,12 +128,11 @@ def run(test, params, env): vm = mig_data.vms[0] session = vm.wait_for_login(timeout=self.login_timeout) - cpuflags.install_cpuflags_util_on_vm(test, vm, install_path, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path, extra_flags="-msse3 -msse2") - cmd = ("%s/cpuflags-test --stressmem %d,%d" % - (os.path.join(install_path, "cpu_flags"), - vm_mem * 4, vm_mem / 2)) + cmd = ("%s --stressmem %d,%d" % + (cpuflags_cmd, vm_mem * 4, vm_mem / 2)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd) diff --git a/qemu/tests/migration_with_speed_measurement.py b/qemu/tests/migration_with_speed_measurement.py index 47c4231..a88d648 100644 --- a/qemu/tests/migration_with_speed_measurement.py +++ b/qemu/tests/migration_with_speed_measurement.py @@ -82,14 +82,13 @@ def run(test, params, env): try: # Reboot the VM in the background - cpuflags.install_cpuflags_util_on_vm(test, vm, install_path, + cpuflags_cmd = cpuflags.install_cpuflags_util_on_vm(test, vm, install_path, extra_flags="-msse3 -msse2") vm.monitor.migrate_set_speed(mig_speed) - cmd = ("%s/cpuflags-test --stressmem %d,%d" % - (os.path.join(install_path, "cpu_flags"), - vm_mem * 4, vm_mem / 2)) + cmd = ("%s --stressmem %d,%d" % + (cpuflags_cmd, vm_mem * 4, vm_mem / 2)) logging.debug("Sending command: %s" % (cmd)) session.sendline(cmd)