summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Kanavin <alex.kanavin@gmail.com>2023-11-29 14:24:24 +0100
committerKhem Raj <raj.khem@gmail.com>2023-11-29 07:35:36 -0800
commita4e9bec26db2a53e8bfc114e406f153f60ad6340 (patch)
treeca286458fbc1460e3b7a453fd57e6f38fc9b8b7f
parent3991d85383b466a4948283cfd0167213a1dd2311 (diff)
downloadmeta-openembedded-a4e9bec26db2a53e8bfc114e406f153f60ad6340.tar.gz
fio: revert "fio: upgrade 3.32 -> 2022"
This was a bogus update (from AUH I believe) and should be reverted. Signed-off-by: Alexander Kanavin <alex@linutronix.de> Signed-off-by: Khem Raj <raj.khem@gmail.com>
Diffstat
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0001-Fio-3.31.patch23
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0002-lib-rand-Enhance-__fill_random_buf-using-the-multi-r.patch136
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0003-lib-rand-get-rid-of-unused-MAX_SEED_BUCKETS.patch31
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0004-ioengines-merge-filecreate-filestat-filedelete-engin.patch800
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0005-engines-http-Add-storage-class-option-for-s3.patch87
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0006-engines-http-Add-s3-crypto-options-for-s3.patch246
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0007-doc-Add-usage-and-example-about-s3-storage-class-and.patch155
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0008-README-link-to-GitHub-releases-for-Windows.patch33
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0009-engines-xnvme-fix-segfault-issue-with-xnvme-ioengine.patch54
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0010-doc-update-fio-doc-for-xnvme-engine.patch168
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0011-test-add-latency-test-using-posixaio-ioengine.patch55
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0012-test-fix-hash-for-t0016.patch33
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0013-doc-get-rid-of-trailing-whitespace.patch82
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0014-doc-clarify-that-I-O-errors-may-go-unnoticed-without.patch54
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0015-Revert-Minor-style-fixups.patch41
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0016-Revert-Fix-multithread-issues-when-operating-on-a-si.patch141
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0017-Add-wait-for-handling-SIGBREAK.patch59
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0018-engines-io_uring-pass-back-correct-error-value-when-.patch58
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0019-Enable-CPU-affinity-support-on-Android.patch64
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0020-io_uring-Replace-pthread_self-with-s-tid.patch41
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0021-engines-io_uring-delete-debug-code.patch37
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0022-t-io_uring-prep-for-including-engines-nvme.h-in-t-io.patch72
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0023-t-io_uring-add-support-for-async-passthru.patch379
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0024-t-io_uring-fix-64-bit-cast-on-32-bit-archs.patch37
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0025-test-add-basic-test-for-io_uring-ioengine.patch91
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0026-t-io_uring-remove-duplicate-definition-of-gettid.patch59
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0027-test-add-some-tests-for-seq-and-rand-offsets.patch157
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0028-test-use-Ubuntu-22.04-for-64-bit-tests.patch72
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0029-test-get-32-bit-Ubuntu-22.04-build-working.patch79
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0030-test-add-tests-for-lfsr-and-norandommap.patch143
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0031-backend-revert-bad-memory-leak-fix.patch39
-rw-r--r--meta-oe/recipes-benchmark/fio/fio/0032-Fio-3.32.patch23
-rw-r--r--meta-oe/recipes-benchmark/fio/fio_2022.bb77
-rw-r--r--meta-oe/recipes-benchmark/fio/fio_3.32.bb44
34 files changed, 44 insertions, 3626 deletions
diff --git a/meta-oe/recipes-benchmark/fio/fio/0001-Fio-3.31.patch b/meta-oe/recipes-benchmark/fio/fio/0001-Fio-3.31.patch
deleted file mode 100644
index 9b8b7d02e..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0001-Fio-3.31.patch
+++ /dev/null
@@ -1,23 +0,0 @@
1From 6cafe8445fd1e04e5f7d67bbc73029a538d1b253 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Tue, 9 Aug 2022 14:41:25 -0600
4Subject: [PATCH] Fio 3.31
5
6Signed-off-by: Jens Axboe <axboe@kernel.dk>
7---
8 FIO-VERSION-GEN | 2 +-
9 1 file changed, 1 insertion(+), 1 deletion(-)
10
11diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN
12index fa64f50f..72630dd0 100755
13--- a/FIO-VERSION-GEN
14+++ b/FIO-VERSION-GEN
15@@ -1,7 +1,7 @@
16 #!/bin/sh
17
18 GVF=FIO-VERSION-FILE
19-DEF_VER=fio-3.30
20+DEF_VER=fio-3.31
21
22 LF='
23 '
diff --git a/meta-oe/recipes-benchmark/fio/fio/0002-lib-rand-Enhance-__fill_random_buf-using-the-multi-r.patch b/meta-oe/recipes-benchmark/fio/fio/0002-lib-rand-Enhance-__fill_random_buf-using-the-multi-r.patch
deleted file mode 100644
index 16506566c..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0002-lib-rand-Enhance-__fill_random_buf-using-the-multi-r.patch
+++ /dev/null
@@ -1,136 +0,0 @@
1From 40ba7a05de6a08cfd382b116f76dbeaa7237df45 Mon Sep 17 00:00:00 2001
2From: Sungup Moon <sungup.moon@samsung.com>
3Date: Mon, 8 Aug 2022 17:21:46 +0900
4Subject: [PATCH] lib/rand: Enhance __fill_random_buf using the multi random
5 seed
6
7The __fill_random_buf fills a buffer using the random 8byte integer to
8write. But, this mechanism is depend on the CPU performance and could
9not reach the max performance on the PCIe Gen5 devices. I have tested
10128KB single worker sequential write on PCIe Gen5 NVMe, but it cannot
11reach write throughput 6.0GB/s.
12
13So, I have reviewed the __fill_random_buf and focused the multiplier
14dependency to generate the random number. So, I have changed
15__fill_random_buf using the multiple-random-seed to reduce the
16dependencies in the small data filling loop.
17
18I'll attach detail analysis result in the PR of this branch.
19
20Signed-off-by: Sungup Moon <sungup.moon@samsung.com>
21---
22 configure | 17 +++++++++++++++++
23 lib/rand.c | 37 ++++++++++++++++++++++++++++++++++++-
24 2 files changed, 53 insertions(+), 1 deletion(-)
25
26diff --git a/configure b/configure
27index 36450df8..a2b9bd4c 100755
28--- a/configure
29+++ b/configure
30@@ -116,6 +116,10 @@ has() {
31 type "$1" >/dev/null 2>&1
32 }
33
34+num() {
35+ echo "$1" | grep -P -q "^[0-9]+$"
36+}
37+
38 check_define() {
39 cat > $TMPC <<EOF
40 #if !defined($1)
41@@ -174,6 +178,7 @@ libnfs=""
42 xnvme=""
43 libzbc=""
44 dfs=""
45+seed_buckets=""
46 dynamic_engines="no"
47 prefix=/usr/local
48
49@@ -255,6 +260,8 @@ for opt do
50 ;;
51 --enable-asan) asan="yes"
52 ;;
53+ --seed-buckets=*) seed_buckets="$optarg"
54+ ;;
55 --help)
56 show_help="yes"
57 ;;
58@@ -302,6 +309,7 @@ if test "$show_help" = "yes" ; then
59 echo "--dynamic-libengines Lib-based ioengines as dynamic libraries"
60 echo "--disable-dfs Disable DAOS File System support even if found"
61 echo "--enable-asan Enable address sanitizer"
62+ echo "--seed-buckets= Number of seed buckets for the refill-buffer"
63 exit $exit_val
64 fi
65
66@@ -3273,6 +3281,15 @@ if test "$disable_tcmalloc" != "yes"; then
67 fi
68 fi
69 print_config "TCMalloc support" "$tcmalloc"
70+if ! num "$seed_buckets"; then
71+ seed_buckets=4
72+elif test "$seed_buckets" -lt 2; then
73+ seed_buckets=2
74+elif test "$seed_buckets" -gt 16; then
75+ seed_buckets=16
76+fi
77+echo "#define CONFIG_SEED_BUCKETS $seed_buckets" >> $config_host_h
78+print_config "seed_buckets" "$seed_buckets"
79
80 echo "LIBS+=$LIBS" >> $config_host_mak
81 echo "GFIO_LIBS+=$GFIO_LIBS" >> $config_host_mak
82diff --git a/lib/rand.c b/lib/rand.c
83index 1e669116..1ce4a849 100644
84--- a/lib/rand.c
85+++ b/lib/rand.c
86@@ -95,7 +95,7 @@ void init_rand_seed(struct frand_state *state, uint64_t seed, bool use64)
87 __init_rand64(&state->state64, seed);
88 }
89
90-void __fill_random_buf(void *buf, unsigned int len, uint64_t seed)
91+void __fill_random_buf_small(void *buf, unsigned int len, uint64_t seed)
92 {
93 uint64_t *b = buf;
94 uint64_t *e = b + len / sizeof(*b);
95@@ -110,6 +110,41 @@ void __fill_random_buf(void *buf, unsigned int len, uint64_t seed)
96 __builtin_memcpy(e, &seed, rest);
97 }
98
99+void __fill_random_buf(void *buf, unsigned int len, uint64_t seed)
100+{
101+#define MAX_SEED_BUCKETS 16
102+ static uint64_t prime[MAX_SEED_BUCKETS] = {1, 2, 3, 5,
103+ 7, 11, 13, 17,
104+ 19, 23, 29, 31,
105+ 37, 41, 43, 47};
106+
107+ uint64_t *b, *e, s[CONFIG_SEED_BUCKETS];
108+ unsigned int rest;
109+ int p;
110+
111+ /*
112+ * Calculate the max index which is multiples of the seed buckets.
113+ */
114+ rest = (len / sizeof(*b) / CONFIG_SEED_BUCKETS) * CONFIG_SEED_BUCKETS;
115+
116+ b = buf;
117+ e = b + rest;
118+
119+ rest = len - (rest * sizeof(*b));
120+
121+ for (p = 0; p < CONFIG_SEED_BUCKETS; p++)
122+ s[p] = seed * prime[p];
123+
124+ for (; b != e; b += CONFIG_SEED_BUCKETS) {
125+ for (p = 0; p < CONFIG_SEED_BUCKETS; ++p) {
126+ b[p] = s[p];
127+ s[p] = __hash_u64(s[p]);
128+ }
129+ }
130+
131+ __fill_random_buf_small(b, rest, s[0]);
132+}
133+
134 uint64_t fill_random_buf(struct frand_state *fs, void *buf,
135 unsigned int len)
136 {
diff --git a/meta-oe/recipes-benchmark/fio/fio/0003-lib-rand-get-rid-of-unused-MAX_SEED_BUCKETS.patch b/meta-oe/recipes-benchmark/fio/fio/0003-lib-rand-get-rid-of-unused-MAX_SEED_BUCKETS.patch
deleted file mode 100644
index fc4af0a51..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0003-lib-rand-get-rid-of-unused-MAX_SEED_BUCKETS.patch
+++ /dev/null
@@ -1,31 +0,0 @@
1From f4dd3f2ad435a75862ad3f34a661b169f72c7885 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Wed, 10 Aug 2022 09:51:49 -0600
4Subject: [PATCH] lib/rand: get rid of unused MAX_SEED_BUCKETS
5
6It's only used to size the array, we don't need it.
7
8Signed-off-by: Jens Axboe <axboe@kernel.dk>
9---
10 lib/rand.c | 8 ++------
11 1 file changed, 2 insertions(+), 6 deletions(-)
12
13diff --git a/lib/rand.c b/lib/rand.c
14index 1ce4a849..0e787a62 100644
15--- a/lib/rand.c
16+++ b/lib/rand.c
17@@ -112,12 +112,8 @@ void __fill_random_buf_small(void *buf, unsigned int len, uint64_t seed)
18
19 void __fill_random_buf(void *buf, unsigned int len, uint64_t seed)
20 {
21-#define MAX_SEED_BUCKETS 16
22- static uint64_t prime[MAX_SEED_BUCKETS] = {1, 2, 3, 5,
23- 7, 11, 13, 17,
24- 19, 23, 29, 31,
25- 37, 41, 43, 47};
26-
27+ static uint64_t prime[] = {1, 2, 3, 5, 7, 11, 13, 17,
28+ 19, 23, 29, 31, 37, 41, 43, 47};
29 uint64_t *b, *e, s[CONFIG_SEED_BUCKETS];
30 unsigned int rest;
31 int p;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0004-ioengines-merge-filecreate-filestat-filedelete-engin.patch b/meta-oe/recipes-benchmark/fio/fio/0004-ioengines-merge-filecreate-filestat-filedelete-engin.patch
deleted file mode 100644
index 1b59ad304..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0004-ioengines-merge-filecreate-filestat-filedelete-engin.patch
+++ /dev/null
@@ -1,800 +0,0 @@
1From 1cfbaff9806f17c2afbabe79c1c87b96eba7f35a Mon Sep 17 00:00:00 2001
2From: "Friendy.Su@sony.com" <Friendy.Su@sony.com>
3Date: Mon, 8 Aug 2022 08:35:50 +0000
4Subject: [PATCH] ioengines: merge filecreate, filestat, filedelete engines to
5 fileoperations.c
6
7file operation engines have similar structure, implement them
8in one file.
9
10Signed-off-by: friendy-su <friendy.su@sony.com>
11---
12 Makefile | 2 +-
13 engines/filecreate.c | 118 ---------------
14 engines/filedelete.c | 115 --------------
15 engines/fileoperations.c | 318 +++++++++++++++++++++++++++++++++++++++
16 engines/filestat.c | 190 -----------------------
17 5 files changed, 319 insertions(+), 424 deletions(-)
18 delete mode 100644 engines/filecreate.c
19 delete mode 100644 engines/filedelete.c
20 create mode 100644 engines/fileoperations.c
21 delete mode 100644 engines/filestat.c
22
23diff --git a/Makefile b/Makefile
24index 188a74d7..634d2c93 100644
25--- a/Makefile
26+++ b/Makefile
27@@ -56,7 +56,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
28 pshared.c options.c \
29 smalloc.c filehash.c profile.c debug.c engines/cpu.c \
30 engines/mmap.c engines/sync.c engines/null.c engines/net.c \
31- engines/ftruncate.c engines/filecreate.c engines/filestat.c engines/filedelete.c \
32+ engines/ftruncate.c engines/fileoperations.c \
33 engines/exec.c \
34 server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \
35 gettime-thread.c helpers.c json.c idletime.c td_error.c \
36diff --git a/engines/filecreate.c b/engines/filecreate.c
37deleted file mode 100644
38index 7884752d..00000000
39--- a/engines/filecreate.c
40+++ /dev/null
41@@ -1,118 +0,0 @@
42-/*
43- * filecreate engine
44- *
45- * IO engine that doesn't do any IO, just creates files and tracks the latency
46- * of the file creation.
47- */
48-#include <stdio.h>
49-#include <fcntl.h>
50-#include <errno.h>
51-
52-#include "../fio.h"
53-
54-struct fc_data {
55- enum fio_ddir stat_ddir;
56-};
57-
58-static int open_file(struct thread_data *td, struct fio_file *f)
59-{
60- struct timespec start;
61- int do_lat = !td->o.disable_lat;
62-
63- dprint(FD_FILE, "fd open %s\n", f->file_name);
64-
65- if (f->filetype != FIO_TYPE_FILE) {
66- log_err("fio: only files are supported\n");
67- return 1;
68- }
69- if (!strcmp(f->file_name, "-")) {
70- log_err("fio: can't read/write to stdin/out\n");
71- return 1;
72- }
73-
74- if (do_lat)
75- fio_gettime(&start, NULL);
76-
77- f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
78-
79- if (f->fd == -1) {
80- char buf[FIO_VERROR_SIZE];
81- int e = errno;
82-
83- snprintf(buf, sizeof(buf), "open(%s)", f->file_name);
84- td_verror(td, e, buf);
85- return 1;
86- }
87-
88- if (do_lat) {
89- struct fc_data *data = td->io_ops_data;
90- uint64_t nsec;
91-
92- nsec = ntime_since_now(&start);
93- add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
94- }
95-
96- return 0;
97-}
98-
99-static enum fio_q_status queue_io(struct thread_data *td,
100- struct io_u fio_unused *io_u)
101-{
102- return FIO_Q_COMPLETED;
103-}
104-
105-/*
106- * Ensure that we at least have a block size worth of IO to do for each
107- * file. If the job file has td->o.size < nr_files * block_size, then
108- * fio won't do anything.
109- */
110-static int get_file_size(struct thread_data *td, struct fio_file *f)
111-{
112- f->real_file_size = td_min_bs(td);
113- return 0;
114-}
115-
116-static int init(struct thread_data *td)
117-{
118- struct fc_data *data;
119-
120- data = calloc(1, sizeof(*data));
121-
122- if (td_read(td))
123- data->stat_ddir = DDIR_READ;
124- else if (td_write(td))
125- data->stat_ddir = DDIR_WRITE;
126-
127- td->io_ops_data = data;
128- return 0;
129-}
130-
131-static void cleanup(struct thread_data *td)
132-{
133- struct fc_data *data = td->io_ops_data;
134-
135- free(data);
136-}
137-
138-static struct ioengine_ops ioengine = {
139- .name = "filecreate",
140- .version = FIO_IOOPS_VERSION,
141- .init = init,
142- .cleanup = cleanup,
143- .queue = queue_io,
144- .get_file_size = get_file_size,
145- .open_file = open_file,
146- .close_file = generic_close_file,
147- .flags = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
148- FIO_NOSTATS | FIO_NOFILEHASH,
149-};
150-
151-static void fio_init fio_filecreate_register(void)
152-{
153- register_ioengine(&ioengine);
154-}
155-
156-static void fio_exit fio_filecreate_unregister(void)
157-{
158- unregister_ioengine(&ioengine);
159-}
160diff --git a/engines/filedelete.c b/engines/filedelete.c
161deleted file mode 100644
162index df388ac9..00000000
163--- a/engines/filedelete.c
164+++ /dev/null
165@@ -1,115 +0,0 @@
166-/*
167- * file delete engine
168- *
169- * IO engine that doesn't do any IO, just delete files and track the latency
170- * of the file deletion.
171- */
172-#include <stdio.h>
173-#include <fcntl.h>
174-#include <errno.h>
175-#include <sys/types.h>
176-#include <unistd.h>
177-#include "../fio.h"
178-
179-struct fc_data {
180- enum fio_ddir stat_ddir;
181-};
182-
183-static int delete_file(struct thread_data *td, struct fio_file *f)
184-{
185- struct timespec start;
186- int do_lat = !td->o.disable_lat;
187- int ret;
188-
189- dprint(FD_FILE, "fd delete %s\n", f->file_name);
190-
191- if (f->filetype != FIO_TYPE_FILE) {
192- log_err("fio: only files are supported\n");
193- return 1;
194- }
195- if (!strcmp(f->file_name, "-")) {
196- log_err("fio: can't read/write to stdin/out\n");
197- return 1;
198- }
199-
200- if (do_lat)
201- fio_gettime(&start, NULL);
202-
203- ret = unlink(f->file_name);
204-
205- if (ret == -1) {
206- char buf[FIO_VERROR_SIZE];
207- int e = errno;
208-
209- snprintf(buf, sizeof(buf), "delete(%s)", f->file_name);
210- td_verror(td, e, buf);
211- return 1;
212- }
213-
214- if (do_lat) {
215- struct fc_data *data = td->io_ops_data;
216- uint64_t nsec;
217-
218- nsec = ntime_since_now(&start);
219- add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
220- }
221-
222- return 0;
223-}
224-
225-
226-static enum fio_q_status queue_io(struct thread_data *td, struct io_u fio_unused *io_u)
227-{
228- return FIO_Q_COMPLETED;
229-}
230-
231-static int init(struct thread_data *td)
232-{
233- struct fc_data *data;
234-
235- data = calloc(1, sizeof(*data));
236-
237- if (td_read(td))
238- data->stat_ddir = DDIR_READ;
239- else if (td_write(td))
240- data->stat_ddir = DDIR_WRITE;
241-
242- td->io_ops_data = data;
243- return 0;
244-}
245-
246-static int delete_invalidate(struct thread_data *td, struct fio_file *f)
247-{
248- /* do nothing because file not opened */
249- return 0;
250-}
251-
252-static void cleanup(struct thread_data *td)
253-{
254- struct fc_data *data = td->io_ops_data;
255-
256- free(data);
257-}
258-
259-static struct ioengine_ops ioengine = {
260- .name = "filedelete",
261- .version = FIO_IOOPS_VERSION,
262- .init = init,
263- .invalidate = delete_invalidate,
264- .cleanup = cleanup,
265- .queue = queue_io,
266- .get_file_size = generic_get_file_size,
267- .open_file = delete_file,
268- .flags = FIO_SYNCIO | FIO_FAKEIO |
269- FIO_NOSTATS | FIO_NOFILEHASH,
270-};
271-
272-static void fio_init fio_filedelete_register(void)
273-{
274- register_ioengine(&ioengine);
275-}
276-
277-static void fio_exit fio_filedelete_unregister(void)
278-{
279- unregister_ioengine(&ioengine);
280-}
281diff --git a/engines/fileoperations.c b/engines/fileoperations.c
282new file mode 100644
283index 00000000..1db60da1
284--- /dev/null
285+++ b/engines/fileoperations.c
286@@ -0,0 +1,318 @@
287+/*
288+ * fileoperations engine
289+ *
290+ * IO engine that doesn't do any IO, just operates files and tracks the latency
291+ * of the file operation.
292+ */
293+#include <stdio.h>
294+#include <stdlib.h>
295+#include <fcntl.h>
296+#include <errno.h>
297+#include <sys/types.h>
298+#include <sys/stat.h>
299+#include <unistd.h>
300+#include "../fio.h"
301+#include "../optgroup.h"
302+#include "../oslib/statx.h"
303+
304+
305+struct fc_data {
306+ enum fio_ddir stat_ddir;
307+};
308+
309+struct filestat_options {
310+ void *pad;
311+ unsigned int stat_type;
312+};
313+
314+enum {
315+ FIO_FILESTAT_STAT = 1,
316+ FIO_FILESTAT_LSTAT = 2,
317+ FIO_FILESTAT_STATX = 3,
318+};
319+
320+static struct fio_option options[] = {
321+ {
322+ .name = "stat_type",
323+ .lname = "stat_type",
324+ .type = FIO_OPT_STR,
325+ .off1 = offsetof(struct filestat_options, stat_type),
326+ .help = "Specify stat system call type to measure lookup/getattr performance",
327+ .def = "stat",
328+ .posval = {
329+ { .ival = "stat",
330+ .oval = FIO_FILESTAT_STAT,
331+ .help = "Use stat(2)",
332+ },
333+ { .ival = "lstat",
334+ .oval = FIO_FILESTAT_LSTAT,
335+ .help = "Use lstat(2)",
336+ },
337+ { .ival = "statx",
338+ .oval = FIO_FILESTAT_STATX,
339+ .help = "Use statx(2) if exists",
340+ },
341+ },
342+ .category = FIO_OPT_C_ENGINE,
343+ .group = FIO_OPT_G_FILESTAT,
344+ },
345+ {
346+ .name = NULL,
347+ },
348+};
349+
350+
351+static int open_file(struct thread_data *td, struct fio_file *f)
352+{
353+ struct timespec start;
354+ int do_lat = !td->o.disable_lat;
355+
356+ dprint(FD_FILE, "fd open %s\n", f->file_name);
357+
358+ if (f->filetype != FIO_TYPE_FILE) {
359+ log_err("fio: only files are supported\n");
360+ return 1;
361+ }
362+ if (!strcmp(f->file_name, "-")) {
363+ log_err("fio: can't read/write to stdin/out\n");
364+ return 1;
365+ }
366+
367+ if (do_lat)
368+ fio_gettime(&start, NULL);
369+
370+ f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
371+
372+ if (f->fd == -1) {
373+ char buf[FIO_VERROR_SIZE];
374+ int e = errno;
375+
376+ snprintf(buf, sizeof(buf), "open(%s)", f->file_name);
377+ td_verror(td, e, buf);
378+ return 1;
379+ }
380+
381+ if (do_lat) {
382+ struct fc_data *data = td->io_ops_data;
383+ uint64_t nsec;
384+
385+ nsec = ntime_since_now(&start);
386+ add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
387+ }
388+
389+ return 0;
390+}
391+
392+static int stat_file(struct thread_data *td, struct fio_file *f)
393+{
394+ struct filestat_options *o = td->eo;
395+ struct timespec start;
396+ int do_lat = !td->o.disable_lat;
397+ struct stat statbuf;
398+#ifndef WIN32
399+ struct statx statxbuf;
400+ char *abspath;
401+#endif
402+ int ret;
403+
404+ dprint(FD_FILE, "fd stat %s\n", f->file_name);
405+
406+ if (f->filetype != FIO_TYPE_FILE) {
407+ log_err("fio: only files are supported\n");
408+ return 1;
409+ }
410+ if (!strcmp(f->file_name, "-")) {
411+ log_err("fio: can't read/write to stdin/out\n");
412+ return 1;
413+ }
414+
415+ if (do_lat)
416+ fio_gettime(&start, NULL);
417+
418+ switch (o->stat_type) {
419+ case FIO_FILESTAT_STAT:
420+ ret = stat(f->file_name, &statbuf);
421+ break;
422+ case FIO_FILESTAT_LSTAT:
423+ ret = lstat(f->file_name, &statbuf);
424+ break;
425+ case FIO_FILESTAT_STATX:
426+#ifndef WIN32
427+ abspath = realpath(f->file_name, NULL);
428+ if (abspath) {
429+ ret = statx(-1, abspath, 0, STATX_ALL, &statxbuf);
430+ free(abspath);
431+ } else
432+ ret = -1;
433+#else
434+ ret = -1;
435+#endif
436+ break;
437+ default:
438+ ret = -1;
439+ break;
440+ }
441+
442+ if (ret == -1) {
443+ char buf[FIO_VERROR_SIZE];
444+ int e = errno;
445+
446+ snprintf(buf, sizeof(buf), "stat(%s) type=%u", f->file_name,
447+ o->stat_type);
448+ td_verror(td, e, buf);
449+ return 1;
450+ }
451+
452+ if (do_lat) {
453+ struct fc_data *data = td->io_ops_data;
454+ uint64_t nsec;
455+
456+ nsec = ntime_since_now(&start);
457+ add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
458+ }
459+
460+ return 0;
461+}
462+
463+
464+static int delete_file(struct thread_data *td, struct fio_file *f)
465+{
466+ struct timespec start;
467+ int do_lat = !td->o.disable_lat;
468+ int ret;
469+
470+ dprint(FD_FILE, "fd delete %s\n", f->file_name);
471+
472+ if (f->filetype != FIO_TYPE_FILE) {
473+ log_err("fio: only files are supported\n");
474+ return 1;
475+ }
476+ if (!strcmp(f->file_name, "-")) {
477+ log_err("fio: can't read/write to stdin/out\n");
478+ return 1;
479+ }
480+
481+ if (do_lat)
482+ fio_gettime(&start, NULL);
483+
484+ ret = unlink(f->file_name);
485+
486+ if (ret == -1) {
487+ char buf[FIO_VERROR_SIZE];
488+ int e = errno;
489+
490+ snprintf(buf, sizeof(buf), "delete(%s)", f->file_name);
491+ td_verror(td, e, buf);
492+ return 1;
493+ }
494+
495+ if (do_lat) {
496+ struct fc_data *data = td->io_ops_data;
497+ uint64_t nsec;
498+
499+ nsec = ntime_since_now(&start);
500+ add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
501+ }
502+
503+ return 0;
504+}
505+
506+static int invalidate_do_nothing(struct thread_data *td, struct fio_file *f)
507+{
508+ /* do nothing because file not opened */
509+ return 0;
510+}
511+
512+static enum fio_q_status queue_io(struct thread_data *td, struct io_u *io_u)
513+{
514+ return FIO_Q_COMPLETED;
515+}
516+
517+/*
518+ * Ensure that we at least have a block size worth of IO to do for each
519+ * file. If the job file has td->o.size < nr_files * block_size, then
520+ * fio won't do anything.
521+ */
522+static int get_file_size(struct thread_data *td, struct fio_file *f)
523+{
524+ f->real_file_size = td_min_bs(td);
525+ return 0;
526+}
527+
528+static int init(struct thread_data *td)
529+{
530+ struct fc_data *data;
531+
532+ data = calloc(1, sizeof(*data));
533+
534+ if (td_read(td))
535+ data->stat_ddir = DDIR_READ;
536+ else if (td_write(td))
537+ data->stat_ddir = DDIR_WRITE;
538+
539+ td->io_ops_data = data;
540+ return 0;
541+}
542+
543+static void cleanup(struct thread_data *td)
544+{
545+ struct fc_data *data = td->io_ops_data;
546+
547+ free(data);
548+}
549+
550+static struct ioengine_ops ioengine_filecreate = {
551+ .name = "filecreate",
552+ .version = FIO_IOOPS_VERSION,
553+ .init = init,
554+ .cleanup = cleanup,
555+ .queue = queue_io,
556+ .get_file_size = get_file_size,
557+ .open_file = open_file,
558+ .close_file = generic_close_file,
559+ .flags = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
560+ FIO_NOSTATS | FIO_NOFILEHASH,
561+};
562+
563+static struct ioengine_ops ioengine_filestat = {
564+ .name = "filestat",
565+ .version = FIO_IOOPS_VERSION,
566+ .init = init,
567+ .cleanup = cleanup,
568+ .queue = queue_io,
569+ .invalidate = invalidate_do_nothing,
570+ .get_file_size = generic_get_file_size,
571+ .open_file = stat_file,
572+ .flags = FIO_SYNCIO | FIO_FAKEIO |
573+ FIO_NOSTATS | FIO_NOFILEHASH,
574+ .options = options,
575+ .option_struct_size = sizeof(struct filestat_options),
576+};
577+
578+static struct ioengine_ops ioengine_filedelete = {
579+ .name = "filedelete",
580+ .version = FIO_IOOPS_VERSION,
581+ .init = init,
582+ .invalidate = invalidate_do_nothing,
583+ .cleanup = cleanup,
584+ .queue = queue_io,
585+ .get_file_size = generic_get_file_size,
586+ .open_file = delete_file,
587+ .flags = FIO_SYNCIO | FIO_FAKEIO |
588+ FIO_NOSTATS | FIO_NOFILEHASH,
589+};
590+
591+
592+static void fio_init fio_fileoperations_register(void)
593+{
594+ register_ioengine(&ioengine_filecreate);
595+ register_ioengine(&ioengine_filestat);
596+ register_ioengine(&ioengine_filedelete);
597+}
598+
599+static void fio_exit fio_fileoperations_unregister(void)
600+{
601+ unregister_ioengine(&ioengine_filecreate);
602+ unregister_ioengine(&ioengine_filestat);
603+ unregister_ioengine(&ioengine_filedelete);
604+}
605diff --git a/engines/filestat.c b/engines/filestat.c
606deleted file mode 100644
607index e587eb54..00000000
608--- a/engines/filestat.c
609+++ /dev/null
610@@ -1,190 +0,0 @@
611-/*
612- * filestat engine
613- *
614- * IO engine that doesn't do any IO, just stat files and tracks the latency
615- * of the file stat.
616- */
617-#include <stdio.h>
618-#include <stdlib.h>
619-#include <fcntl.h>
620-#include <errno.h>
621-#include <sys/types.h>
622-#include <sys/stat.h>
623-#include <unistd.h>
624-#include "../fio.h"
625-#include "../optgroup.h"
626-#include "../oslib/statx.h"
627-
628-struct fc_data {
629- enum fio_ddir stat_ddir;
630-};
631-
632-struct filestat_options {
633- void *pad;
634- unsigned int stat_type;
635-};
636-
637-enum {
638- FIO_FILESTAT_STAT = 1,
639- FIO_FILESTAT_LSTAT = 2,
640- FIO_FILESTAT_STATX = 3,
641-};
642-
643-static struct fio_option options[] = {
644- {
645- .name = "stat_type",
646- .lname = "stat_type",
647- .type = FIO_OPT_STR,
648- .off1 = offsetof(struct filestat_options, stat_type),
649- .help = "Specify stat system call type to measure lookup/getattr performance",
650- .def = "stat",
651- .posval = {
652- { .ival = "stat",
653- .oval = FIO_FILESTAT_STAT,
654- .help = "Use stat(2)",
655- },
656- { .ival = "lstat",
657- .oval = FIO_FILESTAT_LSTAT,
658- .help = "Use lstat(2)",
659- },
660- { .ival = "statx",
661- .oval = FIO_FILESTAT_STATX,
662- .help = "Use statx(2) if exists",
663- },
664- },
665- .category = FIO_OPT_C_ENGINE,
666- .group = FIO_OPT_G_FILESTAT,
667- },
668- {
669- .name = NULL,
670- },
671-};
672-
673-static int stat_file(struct thread_data *td, struct fio_file *f)
674-{
675- struct filestat_options *o = td->eo;
676- struct timespec start;
677- int do_lat = !td->o.disable_lat;
678- struct stat statbuf;
679-#ifndef WIN32
680- struct statx statxbuf;
681- char *abspath;
682-#endif
683- int ret;
684-
685- dprint(FD_FILE, "fd stat %s\n", f->file_name);
686-
687- if (f->filetype != FIO_TYPE_FILE) {
688- log_err("fio: only files are supported\n");
689- return 1;
690- }
691- if (!strcmp(f->file_name, "-")) {
692- log_err("fio: can't read/write to stdin/out\n");
693- return 1;
694- }
695-
696- if (do_lat)
697- fio_gettime(&start, NULL);
698-
699- switch (o->stat_type){
700- case FIO_FILESTAT_STAT:
701- ret = stat(f->file_name, &statbuf);
702- break;
703- case FIO_FILESTAT_LSTAT:
704- ret = lstat(f->file_name, &statbuf);
705- break;
706- case FIO_FILESTAT_STATX:
707-#ifndef WIN32
708- abspath = realpath(f->file_name, NULL);
709- if (abspath) {
710- ret = statx(-1, abspath, 0, STATX_ALL, &statxbuf);
711- free(abspath);
712- } else
713- ret = -1;
714-#else
715- ret = -1;
716-#endif
717- break;
718- default:
719- ret = -1;
720- break;
721- }
722-
723- if (ret == -1) {
724- char buf[FIO_VERROR_SIZE];
725- int e = errno;
726-
727- snprintf(buf, sizeof(buf), "stat(%s) type=%u", f->file_name,
728- o->stat_type);
729- td_verror(td, e, buf);
730- return 1;
731- }
732-
733- if (do_lat) {
734- struct fc_data *data = td->io_ops_data;
735- uint64_t nsec;
736-
737- nsec = ntime_since_now(&start);
738- add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0);
739- }
740-
741- return 0;
742-}
743-
744-static enum fio_q_status queue_io(struct thread_data *td, struct io_u fio_unused *io_u)
745-{
746- return FIO_Q_COMPLETED;
747-}
748-
749-static int init(struct thread_data *td)
750-{
751- struct fc_data *data;
752-
753- data = calloc(1, sizeof(*data));
754-
755- if (td_read(td))
756- data->stat_ddir = DDIR_READ;
757- else if (td_write(td))
758- data->stat_ddir = DDIR_WRITE;
759-
760- td->io_ops_data = data;
761- return 0;
762-}
763-
764-static void cleanup(struct thread_data *td)
765-{
766- struct fc_data *data = td->io_ops_data;
767-
768- free(data);
769-}
770-
771-static int stat_invalidate(struct thread_data *td, struct fio_file *f)
772-{
773- /* do nothing because file not opened */
774- return 0;
775-}
776-
777-static struct ioengine_ops ioengine = {
778- .name = "filestat",
779- .version = FIO_IOOPS_VERSION,
780- .init = init,
781- .cleanup = cleanup,
782- .queue = queue_io,
783- .invalidate = stat_invalidate,
784- .get_file_size = generic_get_file_size,
785- .open_file = stat_file,
786- .flags = FIO_SYNCIO | FIO_FAKEIO |
787- FIO_NOSTATS | FIO_NOFILEHASH,
788- .options = options,
789- .option_struct_size = sizeof(struct filestat_options),
790-};
791-
792-static void fio_init fio_filestat_register(void)
793-{
794- register_ioengine(&ioengine);
795-}
796-
797-static void fio_exit fio_filestat_unregister(void)
798-{
799- unregister_ioengine(&ioengine);
800-}
diff --git a/meta-oe/recipes-benchmark/fio/fio/0005-engines-http-Add-storage-class-option-for-s3.patch b/meta-oe/recipes-benchmark/fio/fio/0005-engines-http-Add-storage-class-option-for-s3.patch
deleted file mode 100644
index 587df1ada..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0005-engines-http-Add-storage-class-option-for-s3.patch
+++ /dev/null
@@ -1,87 +0,0 @@
1From 5b0b5247f0770a89084964274bb951f5a4393299 Mon Sep 17 00:00:00 2001
2From: "Feng, Hualong" <hualong.feng@intel.com>
3Date: Wed, 20 Jul 2022 12:01:35 +0800
4Subject: [PATCH] engines/http: Add storage class option for s3
5
6Amazon S3 offers a range of storage classes that you can choose from
7based on the data access, resiliency, and cost requirements of your
8workloads. (https://aws.amazon.com/s3/storage-classes/)
9
10For example, we have **STANDARD** storage class to test normal
11workload, and have **COLD** storage class to test the workload
12with gzip compression. It is convenient to select which
13storage class to access for different kinds data.
14
15Signed-off-by: Feng, Hualong <hualong.feng@intel.com>
16---
17 engines/http.c | 25 +++++++++++++++++++------
18 1 file changed, 19 insertions(+), 6 deletions(-)
19
20diff --git a/engines/http.c b/engines/http.c
21index 1de9e66c..dbcde287 100644
22--- a/engines/http.c
23+++ b/engines/http.c
24@@ -57,6 +57,7 @@ struct http_options {
25 char *s3_key;
26 char *s3_keyid;
27 char *s3_region;
28+ char *s3_storage_class;
29 char *swift_auth_token;
30 int verbose;
31 unsigned int mode;
32@@ -161,6 +162,16 @@ static struct fio_option options[] = {
33 .category = FIO_OPT_C_ENGINE,
34 .group = FIO_OPT_G_HTTP,
35 },
36+ {
37+ .name = "http_s3_storage_class",
38+ .lname = "S3 Storage class",
39+ .type = FIO_OPT_STR_STORE,
40+ .help = "S3 Storage Class",
41+ .off1 = offsetof(struct http_options, s3_storage_class),
42+ .def = "STANDARD",
43+ .category = FIO_OPT_C_ENGINE,
44+ .group = FIO_OPT_G_HTTP,
45+ },
46 {
47 .name = "http_mode",
48 .lname = "Request mode to use",
49@@ -335,8 +346,8 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
50 char date_iso[32];
51 char method[8];
52 char dkey[128];
53- char creq[512];
54- char sts[256];
55+ char creq[4096];
56+ char sts[512];
57 char s[512];
58 char *uri_encoded = NULL;
59 char *dsha = NULL;
60@@ -373,11 +384,12 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
61 "host:%s\n"
62 "x-amz-content-sha256:%s\n"
63 "x-amz-date:%s\n"
64+ "x-amz-storage-class:%s\n"
65 "\n"
66- "host;x-amz-content-sha256;x-amz-date\n"
67+ "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
68 "%s"
69 , method
70- , uri_encoded, o->host, dsha, date_iso, dsha);
71+ , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
72
73 csha = _gen_hex_sha256(creq, strlen(creq));
74 snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
75@@ -400,9 +412,10 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
76
77 snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
78 slist = curl_slist_append(slist, s);
79-
80+ snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
81+ slist = curl_slist_append(slist, s);
82 snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
83- "SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=%s",
84+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
85 o->s3_keyid, date_short, o->s3_region, signature);
86 slist = curl_slist_append(slist, s);
87
diff --git a/meta-oe/recipes-benchmark/fio/fio/0006-engines-http-Add-s3-crypto-options-for-s3.patch b/meta-oe/recipes-benchmark/fio/fio/0006-engines-http-Add-s3-crypto-options-for-s3.patch
deleted file mode 100644
index d8222c4d1..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0006-engines-http-Add-s3-crypto-options-for-s3.patch
+++ /dev/null
@@ -1,246 +0,0 @@
1From d196fda02eb73958c2acd367de650858c6203420 Mon Sep 17 00:00:00 2001
2From: "Feng, Hualong" <hualong.feng@intel.com>
3Date: Wed, 20 Jul 2022 09:41:35 +0800
4Subject: [PATCH] engines/http: Add s3 crypto options for s3
5
6Server-side encryption is about protecting data at rest.
7(https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html)
8
9When we want to test server-side encryption, we need to specify
10server-side encryption with customer-provided encryption keys (SSE-C).
11The two option **http_s3_sse_customer_key** and
12**http_s3_sse_customer_algorithm** is for server-side encryption.
13
14Signed-off-by: Feng, Hualong <hualong.feng@intel.com>
15---
16 engines/http.c | 163 +++++++++++++++++++++++++++++++++++++++++++------
17 1 file changed, 146 insertions(+), 17 deletions(-)
18
19diff --git a/engines/http.c b/engines/http.c
20index dbcde287..56dc7d1b 100644
21--- a/engines/http.c
22+++ b/engines/http.c
23@@ -57,6 +57,8 @@ struct http_options {
24 char *s3_key;
25 char *s3_keyid;
26 char *s3_region;
27+ char *s3_sse_customer_key;
28+ char *s3_sse_customer_algorithm;
29 char *s3_storage_class;
30 char *swift_auth_token;
31 int verbose;
32@@ -162,6 +164,26 @@ static struct fio_option options[] = {
33 .category = FIO_OPT_C_ENGINE,
34 .group = FIO_OPT_G_HTTP,
35 },
36+ {
37+ .name = "http_s3_sse_customer_key",
38+ .lname = "SSE Customer Key",
39+ .type = FIO_OPT_STR_STORE,
40+ .help = "S3 SSE Customer Key",
41+ .off1 = offsetof(struct http_options, s3_sse_customer_key),
42+ .def = "",
43+ .category = FIO_OPT_C_ENGINE,
44+ .group = FIO_OPT_G_HTTP,
45+ },
46+ {
47+ .name = "http_s3_sse_customer_algorithm",
48+ .lname = "SSE Customer Algorithm",
49+ .type = FIO_OPT_STR_STORE,
50+ .help = "S3 SSE Customer Algorithm",
51+ .off1 = offsetof(struct http_options, s3_sse_customer_algorithm),
52+ .def = "AES256",
53+ .category = FIO_OPT_C_ENGINE,
54+ .group = FIO_OPT_G_HTTP,
55+ },
56 {
57 .name = "http_s3_storage_class",
58 .lname = "S3 Storage class",
59@@ -277,6 +299,54 @@ static char *_gen_hex_md5(const char *p, size_t len)
60 return _conv_hex(hash, MD5_DIGEST_LENGTH);
61 }
62
63+static char *_conv_base64_encode(const unsigned char *p, size_t len)
64+{
65+ char *r, *ret;
66+ int i;
67+ static const char sEncodingTable[] = {
68+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
69+ 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
70+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
71+ 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
72+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
73+ 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
74+ 'w', 'x', 'y', 'z', '0', '1', '2', '3',
75+ '4', '5', '6', '7', '8', '9', '+', '/'
76+ };
77+
78+ size_t out_len = 4 * ((len + 2) / 3);
79+ ret = r = malloc(out_len + 1);
80+
81+ for (i = 0; i < len - 2; i += 3) {
82+ *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
83+ *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
84+ *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2) | ((int) (p[i + 2] & 0xC0) >> 6)];
85+ *r++ = sEncodingTable[p[i + 2] & 0x3F];
86+ }
87+
88+ if (i < len) {
89+ *r++ = sEncodingTable[(p[i] >> 2) & 0x3F];
90+ if (i == (len - 1)) {
91+ *r++ = sEncodingTable[((p[i] & 0x3) << 4)];
92+ *r++ = '=';
93+ } else {
94+ *r++ = sEncodingTable[((p[i] & 0x3) << 4) | ((int) (p[i + 1] & 0xF0) >> 4)];
95+ *r++ = sEncodingTable[((p[i + 1] & 0xF) << 2)];
96+ }
97+ *r++ = '=';
98+ }
99+
100+ ret[out_len]=0;
101+ return ret;
102+}
103+
104+static char *_gen_base64_md5(const unsigned char *p, size_t len)
105+{
106+ unsigned char hash[MD5_DIGEST_LENGTH];
107+ MD5((unsigned char*)p, len, hash);
108+ return _conv_base64_encode(hash, MD5_DIGEST_LENGTH);
109+}
110+
111 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
112 #ifndef CONFIG_HAVE_OPAQUE_HMAC_CTX
113 HMAC_CTX _ctx;
114@@ -356,6 +426,9 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
115 const char *service = "s3";
116 const char *aws = "aws4_request";
117 unsigned char md[SHA256_DIGEST_LENGTH];
118+ unsigned char sse_key[33] = {0};
119+ char *sse_key_base64 = NULL;
120+ char *sse_key_md5_base64 = NULL;
121
122 time_t t = time(NULL);
123 struct tm *gtm = gmtime(&t);
124@@ -364,6 +437,9 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
125 strftime (date_iso, sizeof(date_iso), "%Y%m%dT%H%M%SZ", gtm);
126 uri_encoded = _aws_uriencode(uri);
127
128+ if (o->s3_sse_customer_key != NULL)
129+ strncpy((char*)sse_key, o->s3_sse_customer_key, sizeof(sse_key) - 1);
130+
131 if (op == DDIR_WRITE) {
132 dsha = _gen_hex_sha256(buf, len);
133 sprintf(method, "PUT");
134@@ -377,23 +453,50 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
135 }
136
137 /* Create the canonical request first */
138- snprintf(creq, sizeof(creq),
139- "%s\n"
140- "%s\n"
141- "\n"
142- "host:%s\n"
143- "x-amz-content-sha256:%s\n"
144- "x-amz-date:%s\n"
145- "x-amz-storage-class:%s\n"
146- "\n"
147- "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
148- "%s"
149- , method
150- , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
151+ if (sse_key[0] != '\0') {
152+ sse_key_base64 = _conv_base64_encode(sse_key, sizeof(sse_key) - 1);
153+ sse_key_md5_base64 = _gen_base64_md5(sse_key, sizeof(sse_key) - 1);
154+ snprintf(creq, sizeof(creq),
155+ "%s\n"
156+ "%s\n"
157+ "\n"
158+ "host:%s\n"
159+ "x-amz-content-sha256:%s\n"
160+ "x-amz-date:%s\n"
161+ "x-amz-server-side-encryption-customer-algorithm:%s\n"
162+ "x-amz-server-side-encryption-customer-key:%s\n"
163+ "x-amz-server-side-encryption-customer-key-md5:%s\n"
164+ "x-amz-storage-class:%s\n"
165+ "\n"
166+ "host;x-amz-content-sha256;x-amz-date;"
167+ "x-amz-server-side-encryption-customer-algorithm;"
168+ "x-amz-server-side-encryption-customer-key;"
169+ "x-amz-server-side-encryption-customer-key-md5;"
170+ "x-amz-storage-class\n"
171+ "%s"
172+ , method
173+ , uri_encoded, o->host, dsha, date_iso
174+ , o->s3_sse_customer_algorithm, sse_key_base64
175+ , sse_key_md5_base64, o->s3_storage_class, dsha);
176+ } else {
177+ snprintf(creq, sizeof(creq),
178+ "%s\n"
179+ "%s\n"
180+ "\n"
181+ "host:%s\n"
182+ "x-amz-content-sha256:%s\n"
183+ "x-amz-date:%s\n"
184+ "x-amz-storage-class:%s\n"
185+ "\n"
186+ "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class\n"
187+ "%s"
188+ , method
189+ , uri_encoded, o->host, dsha, date_iso, o->s3_storage_class, dsha);
190+ }
191
192 csha = _gen_hex_sha256(creq, strlen(creq));
193 snprintf(sts, sizeof(sts), "AWS4-HMAC-SHA256\n%s\n%s/%s/%s/%s\n%s",
194- date_iso, date_short, o->s3_region, service, aws, csha);
195+ date_iso, date_short, o->s3_region, service, aws, csha);
196
197 snprintf((char *)dkey, sizeof(dkey), "AWS4%s", o->s3_key);
198 _hmac(md, dkey, strlen(dkey), date_short);
199@@ -412,11 +515,33 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
200
201 snprintf(s, sizeof(s), "x-amz-date: %s", date_iso);
202 slist = curl_slist_append(slist, s);
203+
204+ if (sse_key[0] != '\0') {
205+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-algorithm: %s", o->s3_sse_customer_algorithm);
206+ slist = curl_slist_append(slist, s);
207+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key: %s", sse_key_base64);
208+ slist = curl_slist_append(slist, s);
209+ snprintf(s, sizeof(s), "x-amz-server-side-encryption-customer-key-md5: %s", sse_key_md5_base64);
210+ slist = curl_slist_append(slist, s);
211+ }
212+
213 snprintf(s, sizeof(s), "x-amz-storage-class: %s", o->s3_storage_class);
214 slist = curl_slist_append(slist, s);
215- snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
216- "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
217- o->s3_keyid, date_short, o->s3_region, signature);
218+
219+ if (sse_key[0] != '\0') {
220+ snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
221+ "SignedHeaders=host;x-amz-content-sha256;"
222+ "x-amz-date;x-amz-server-side-encryption-customer-algorithm;"
223+ "x-amz-server-side-encryption-customer-key;"
224+ "x-amz-server-side-encryption-customer-key-md5;"
225+ "x-amz-storage-class,"
226+ "Signature=%s",
227+ o->s3_keyid, date_short, o->s3_region, signature);
228+ } else {
229+ snprintf(s, sizeof(s), "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,"
230+ "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-storage-class,Signature=%s",
231+ o->s3_keyid, date_short, o->s3_region, signature);
232+ }
233 slist = curl_slist_append(slist, s);
234
235 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
236@@ -425,6 +550,10 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
237 free(csha);
238 free(dsha);
239 free(signature);
240+ if (sse_key_base64 != NULL) {
241+ free(sse_key_base64);
242+ free(sse_key_md5_base64);
243+ }
244 }
245
246 static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0007-doc-Add-usage-and-example-about-s3-storage-class-and.patch b/meta-oe/recipes-benchmark/fio/fio/0007-doc-Add-usage-and-example-about-s3-storage-class-and.patch
deleted file mode 100644
index d5815935d..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0007-doc-Add-usage-and-example-about-s3-storage-class-and.patch
+++ /dev/null
@@ -1,155 +0,0 @@
1From 0cad4b6957818937519604b466a2da5b0c395cfe Mon Sep 17 00:00:00 2001
2From: "Feng, Hualong" <hualong.feng@intel.com>
3Date: Thu, 28 Jul 2022 01:47:48 +0000
4Subject: [PATCH] doc: Add usage and example about s3 storage class and crypto
5
6There add option usage about s3 storage class
7`http_s3_storage_class` and
8s3 SSE server side encryption
9`http_s3_sse_customer_key` and
10`http_s3_sse_customer_algorithm`
11
12And add example file in example folder.
13
14Signed-off-by: Feng, Hualong <hualong.feng@intel.com>
15---
16 HOWTO.rst | 14 +++++++++++
17 examples/http-s3-crypto.fio | 38 ++++++++++++++++++++++++++++++
18 examples/http-s3-storage-class.fio | 37 +++++++++++++++++++++++++++++
19 fio.1 | 9 +++++++
20 4 files changed, 98 insertions(+)
21 create mode 100644 examples/http-s3-crypto.fio
22 create mode 100644 examples/http-s3-storage-class.fio
23
24diff --git a/HOWTO.rst b/HOWTO.rst
25index 104cce2d..05fc117f 100644
26--- a/HOWTO.rst
27+++ b/HOWTO.rst
28@@ -2692,6 +2692,20 @@ with the caveat that when used on the command line, they must come after the
29
30 The S3 key/access id.
31
32+.. option:: http_s3_sse_customer_key=str : [http]
33+
34+ The encryption customer key in SSE server side.
35+
36+.. option:: http_s3_sse_customer_algorithm=str : [http]
37+
38+ The encryption customer algorithm in SSE server side.
39+ Default is **AES256**
40+
41+.. option:: http_s3_storage_class=str : [http]
42+
43+ Which storage class to access. User-customizable settings.
44+ Default is **STANDARD**
45+
46 .. option:: http_swift_auth_token=str : [http]
47
48 The Swift auth token. See the example configuration file on how
49diff --git a/examples/http-s3-crypto.fio b/examples/http-s3-crypto.fio
50new file mode 100644
51index 00000000..2403746e
52--- /dev/null
53+++ b/examples/http-s3-crypto.fio
54@@ -0,0 +1,38 @@
55+# Example test for the HTTP engine's S3 support against Amazon AWS.
56+# Obviously, you have to adjust the S3 credentials; for this example,
57+# they're passed in via the environment.
58+# And you can set the SSE Customer Key and Algorithm to test Server
59+# Side Encryption.
60+#
61+
62+[global]
63+ioengine=http
64+name=test
65+direct=1
66+filename=/larsmb-fio-test/object
67+http_verbose=0
68+https=on
69+http_mode=s3
70+http_s3_key=${S3_KEY}
71+http_s3_keyid=${S3_ID}
72+http_host=s3.eu-central-1.amazonaws.com
73+http_s3_region=eu-central-1
74+http_s3_sse_customer_key=${SSE_KEY}
75+http_s3_sse_customer_algorithm=AES256
76+group_reporting
77+
78+# With verify, this both writes and reads the object
79+[create]
80+rw=write
81+bs=4k
82+size=64k
83+io_size=4k
84+verify=sha256
85+
86+[trim]
87+stonewall
88+rw=trim
89+bs=4k
90+size=64k
91+io_size=4k
92+
93diff --git a/examples/http-s3-storage-class.fio b/examples/http-s3-storage-class.fio
94new file mode 100644
95index 00000000..9ee23837
96--- /dev/null
97+++ b/examples/http-s3-storage-class.fio
98@@ -0,0 +1,37 @@
99+# Example test for the HTTP engine's S3 support against Amazon AWS.
100+# Obviously, you have to adjust the S3 credentials; for this example,
101+# they're passed in via the environment.
102+# And here add storage class parameter, you can set normal test for
103+# STANDARD and compression test for another storage class.
104+#
105+
106+[global]
107+ioengine=http
108+name=test
109+direct=1
110+filename=/larsmb-fio-test/object
111+http_verbose=0
112+https=on
113+http_mode=s3
114+http_s3_key=${S3_KEY}
115+http_s3_keyid=${S3_ID}
116+http_host=s3.eu-central-1.amazonaws.com
117+http_s3_region=eu-central-1
118+http_s3_storage_class=${STORAGE_CLASS}
119+group_reporting
120+
121+# With verify, this both writes and reads the object
122+[create]
123+rw=write
124+bs=4k
125+size=64k
126+io_size=4k
127+verify=sha256
128+
129+[trim]
130+stonewall
131+rw=trim
132+bs=4k
133+size=64k
134+io_size=4k
135+
136diff --git a/fio.1 b/fio.1
137index ce9bf3ef..6630525f 100644
138--- a/fio.1
139+++ b/fio.1
140@@ -2308,6 +2308,15 @@ The S3 secret key.
141 .BI (http)http_s3_keyid \fR=\fPstr
142 The S3 key/access id.
143 .TP
144+.BI (http)http_s3_sse_customer_key \fR=\fPstr
145+The encryption customer key in SSE server side.
146+.TP
147+.BI (http)http_s3_sse_customer_algorithm \fR=\fPstr
148+The encryption customer algorithm in SSE server side. Default is \fBAES256\fR
149+.TP
150+.BI (http)http_s3_storage_class \fR=\fPstr
151+Which storage class to access. User-customizable settings. Default is \fBSTANDARD\fR
152+.TP
153 .BI (http)http_swift_auth_token \fR=\fPstr
154 The Swift auth token. See the example configuration file on how to
155 retrieve this.
diff --git a/meta-oe/recipes-benchmark/fio/fio/0008-README-link-to-GitHub-releases-for-Windows.patch b/meta-oe/recipes-benchmark/fio/fio/0008-README-link-to-GitHub-releases-for-Windows.patch
deleted file mode 100644
index 3f7ba4ccb..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0008-README-link-to-GitHub-releases-for-Windows.patch
+++ /dev/null
@@ -1,33 +0,0 @@
1From 6809d81b2a9b854697c65e0d69455a39d4497a6b Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 15 Aug 2022 10:37:57 -0400
4Subject: [PATCH] README: link to GitHub releases for Windows
5
6Note that Windows installers are now available as releases on GitHub.
7
8Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
9---
10 README.rst | 10 ++++++----
11 1 file changed, 6 insertions(+), 4 deletions(-)
12
13diff --git a/README.rst b/README.rst
14index 67420903..79582dea 100644
15--- a/README.rst
16+++ b/README.rst
17@@ -123,10 +123,12 @@ Solaris:
18 ``pkgutil -i fio``.
19
20 Windows:
21- Rebecca Cran <rebecca@bsdio.com> has fio packages for Windows at
22- https://bsdio.com/fio/ . The latest builds for Windows can also
23- be grabbed from https://ci.appveyor.com/project/axboe/fio by clicking
24- the latest x86 or x64 build, then selecting the ARTIFACTS tab.
25+ Beginning with fio 3.31 Windows installers are available on GitHub at
26+ https://github.com/axboe/fio/releases. Rebecca Cran
27+ <rebecca@bsdio.com> has fio packages for Windows at
28+ https://bsdio.com/fio/ . The latest builds for Windows can also be
29+ grabbed from https://ci.appveyor.com/project/axboe/fio by clicking the
30+ latest x86 or x64 build and then selecting the Artifacts tab.
31
32 BSDs:
33 Packages for BSDs may be available from their binary package repositories.
diff --git a/meta-oe/recipes-benchmark/fio/fio/0009-engines-xnvme-fix-segfault-issue-with-xnvme-ioengine.patch b/meta-oe/recipes-benchmark/fio/fio/0009-engines-xnvme-fix-segfault-issue-with-xnvme-ioengine.patch
deleted file mode 100644
index 6e322b8f9..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0009-engines-xnvme-fix-segfault-issue-with-xnvme-ioengine.patch
+++ /dev/null
@@ -1,54 +0,0 @@
1From d5aac3401e180f3d4ff3db04ebb4e3165b975987 Mon Sep 17 00:00:00 2001
2From: Ankit Kumar <ankit.kumar@samsung.com>
3Date: Tue, 16 Aug 2022 11:08:20 +0530
4Subject: [PATCH] engines/xnvme: fix segfault issue with xnvme ioengine
5
6fix segfault when xnvme ioengine is called without thread=1.
7The segfault happens because td->io_ops_data is accessed at
8two locations xnvme_fioe_cleanup and xnvme_fioe_iomem_free,
9during the error handling call.
10
11Signed-off-by: Ankit Kumar <ankit.kumar@samsung.com>
12Link: https://lore.kernel.org/r/20220816053821.440-2-ankit.kumar@samsung.com
13Signed-off-by: Jens Axboe <axboe@kernel.dk>
14---
15 engines/xnvme.c | 17 ++++++++++++++---
16 1 file changed, 14 insertions(+), 3 deletions(-)
17
18diff --git a/engines/xnvme.c b/engines/xnvme.c
19index c11b33a8..d8647481 100644
20--- a/engines/xnvme.c
21+++ b/engines/xnvme.c
22@@ -205,9 +205,14 @@ static void _dev_close(struct thread_data *td, struct xnvme_fioe_fwrap *fwrap)
23
24 static void xnvme_fioe_cleanup(struct thread_data *td)
25 {
26- struct xnvme_fioe_data *xd = td->io_ops_data;
27+ struct xnvme_fioe_data *xd = NULL;
28 int err;
29
30+ if (!td->io_ops_data)
31+ return;
32+
33+ xd = td->io_ops_data;
34+
35 err = pthread_mutex_lock(&g_serialize);
36 if (err)
37 log_err("ioeng->cleanup(): pthread_mutex_lock(), err(%d)\n", err);
38@@ -367,8 +372,14 @@ static int xnvme_fioe_iomem_alloc(struct thread_data *td, size_t total_mem)
39 /* NOTE: using the first device for buffer-allocators) */
40 static void xnvme_fioe_iomem_free(struct thread_data *td)
41 {
42- struct xnvme_fioe_data *xd = td->io_ops_data;
43- struct xnvme_fioe_fwrap *fwrap = &xd->files[0];
44+ struct xnvme_fioe_data *xd = NULL;
45+ struct xnvme_fioe_fwrap *fwrap = NULL;
46+
47+ if (!td->io_ops_data)
48+ return;
49+
50+ xd = td->io_ops_data;
51+ fwrap = &xd->files[0];
52
53 if (!fwrap->dev) {
54 log_err("ioeng->iomem_free(): failed no dev-handle\n");
diff --git a/meta-oe/recipes-benchmark/fio/fio/0010-doc-update-fio-doc-for-xnvme-engine.patch b/meta-oe/recipes-benchmark/fio/fio/0010-doc-update-fio-doc-for-xnvme-engine.patch
deleted file mode 100644
index 6c85cfc20..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0010-doc-update-fio-doc-for-xnvme-engine.patch
+++ /dev/null
@@ -1,168 +0,0 @@
1From 8e318fd65ba5c0f6ce82860984bc8d69a7843f97 Mon Sep 17 00:00:00 2001
2From: Ankit Kumar <ankit.kumar@samsung.com>
3Date: Tue, 16 Aug 2022 11:08:21 +0530
4Subject: [PATCH] doc: update fio doc for xnvme engine
5
6- Elaborate about the various sync, async and admin
7 interfaces.
8- add missing io_uring_cmd async backend entry.
9- xnvme ioengine doesn't support file stat.
10
11Signed-off-by: Ankit Kumar <ankit.kumar@samsung.com>
12Link: https://lore.kernel.org/r/20220816053821.440-3-ankit.kumar@samsung.com
13Signed-off-by: Jens Axboe <axboe@kernel.dk>
14---
15 HOWTO.rst | 37 ++++++++++++++++++++++++++-----------
16 fio.1 | 34 +++++++++++++++++++++-------------
17 2 files changed, 47 insertions(+), 24 deletions(-)
18
19diff --git a/HOWTO.rst b/HOWTO.rst
20index 05fc117f..b2750b56 100644
21--- a/HOWTO.rst
22+++ b/HOWTO.rst
23@@ -2780,41 +2780,56 @@ with the caveat that when used on the command line, they must come after the
24 Select the xnvme async command interface. This can take these values.
25
26 **emu**
27- This is default and used to emulate asynchronous I/O.
28+ This is default and use to emulate asynchronous I/O by using a
29+ single thread to create a queue pair on top of a synchronous
30+ I/O interface using the NVMe driver IOCTL.
31 **thrpool**
32- Use thread pool for Asynchronous I/O.
33+ Emulate an asynchronous I/O interface with a pool of userspace
34+ threads on top of a synchronous I/O interface using the NVMe
35+ driver IOCTL. By default four threads are used.
36 **io_uring**
37- Use Linux io_uring/liburing for Asynchronous I/O.
38+ Linux native asynchronous I/O interface which supports both
39+ direct and buffered I/O.
40+ **io_uring_cmd**
41+ Fast Linux native asynchronous I/O interface for NVMe pass
42+ through commands. This only works with NVMe character device
43+ (/dev/ngXnY).
44 **libaio**
45 Use Linux aio for Asynchronous I/O.
46 **posix**
47- Use POSIX aio for Asynchronous I/O.
48+ Use the posix asynchronous I/O interface to perform one or
49+ more I/O operations asynchronously.
50 **nil**
51- Use nil-io; For introspective perf. evaluation
52+ Do not transfer any data; just pretend to. This is mainly used
53+ for introspective performance evaluation.
54
55 .. option:: xnvme_sync=str : [xnvme]
56
57 Select the xnvme synchronous command interface. This can take these values.
58
59 **nvme**
60- This is default and uses Linux NVMe Driver ioctl() for synchronous I/O.
61+ This is default and uses Linux NVMe Driver ioctl() for
62+ synchronous I/O.
63 **psync**
64- Use pread()/write() for synchronous I/O.
65+ This supports regular as well as vectored pread() and pwrite()
66+ commands.
67+ **block**
68+ This is the same as psync except that it also supports zone
69+ management commands using Linux block layer IOCTLs.
70
71 .. option:: xnvme_admin=str : [xnvme]
72
73 Select the xnvme admin command interface. This can take these values.
74
75 **nvme**
76- This is default and uses linux NVMe Driver ioctl() for admin commands.
77+ This is default and uses linux NVMe Driver ioctl() for admin
78+ commands.
79 **block**
80 Use Linux Block Layer ioctl() and sysfs for admin commands.
81- **file_as_ns**
82- Use file-stat to construct NVMe idfy responses.
83
84 .. option:: xnvme_dev_nsid=int : [xnvme]
85
86- xnvme namespace identifier, for userspace NVMe driver.
87+ xnvme namespace identifier for userspace NVMe driver, such as SPDK.
88
89 .. option:: xnvme_iovec=int : [xnvme]
90
91diff --git a/fio.1 b/fio.1
92index 6630525f..f3f3dc5d 100644
93--- a/fio.1
94+++ b/fio.1
95@@ -2530,22 +2530,29 @@ Select the xnvme async command interface. This can take these values.
96 .RS
97 .TP
98 .B emu
99-This is default and used to emulate asynchronous I/O
100+This is default and use to emulate asynchronous I/O by using a single thread to
101+create a queue pair on top of a synchronous I/O interface using the NVMe driver
102+IOCTL.
103 .TP
104 .BI thrpool
105-Use thread pool for Asynchronous I/O
106+Emulate an asynchronous I/O interface with a pool of userspace threads on top
107+of a synchronous I/O interface using the NVMe driver IOCTL. By default four
108+threads are used.
109 .TP
110 .BI io_uring
111-Use Linux io_uring/liburing for Asynchronous I/O
112+Linux native asynchronous I/O interface which supports both direct and buffered
113+I/O.
114 .TP
115 .BI libaio
116 Use Linux aio for Asynchronous I/O
117 .TP
118 .BI posix
119-Use POSIX aio for Asynchronous I/O
120+Use the posix asynchronous I/O interface to perform one or more I/O operations
121+asynchronously.
122 .TP
123 .BI nil
124-Use nil-io; For introspective perf. evaluation
125+Do not transfer any data; just pretend to. This is mainly used for
126+introspective performance evaluation.
127 .RE
128 .RE
129 .TP
130@@ -2555,10 +2562,14 @@ Select the xnvme synchronous command interface. This can take these values.
131 .RS
132 .TP
133 .B nvme
134-This is default and uses Linux NVMe Driver ioctl() for synchronous I/O
135+This is default and uses Linux NVMe Driver ioctl() for synchronous I/O.
136 .TP
137 .BI psync
138-Use pread()/write() for synchronous I/O
139+This supports regular as well as vectored pread() and pwrite() commands.
140+.TP
141+.BI block
142+This is the same as psync except that it also supports zone management
143+commands using Linux block layer IOCTLs.
144 .RE
145 .RE
146 .TP
147@@ -2568,18 +2579,15 @@ Select the xnvme admin command interface. This can take these values.
148 .RS
149 .TP
150 .B nvme
151-This is default and uses Linux NVMe Driver ioctl() for admin commands
152+This is default and uses Linux NVMe Driver ioctl() for admin commands.
153 .TP
154 .BI block
155-Use Linux Block Layer ioctl() and sysfs for admin commands
156-.TP
157-.BI file_as_ns
158-Use file-stat as to construct NVMe idfy responses
159+Use Linux Block Layer ioctl() and sysfs for admin commands.
160 .RE
161 .RE
162 .TP
163 .BI (xnvme)xnvme_dev_nsid\fR=\fPint
164-xnvme namespace identifier, for userspace NVMe driver.
165+xnvme namespace identifier for userspace NVMe driver such as SPDK.
166 .TP
167 .BI (xnvme)xnvme_iovec
168 If this option is set, xnvme will use vectored read/write commands.
diff --git a/meta-oe/recipes-benchmark/fio/fio/0011-test-add-latency-test-using-posixaio-ioengine.patch b/meta-oe/recipes-benchmark/fio/fio/0011-test-add-latency-test-using-posixaio-ioengine.patch
deleted file mode 100644
index d86ac11ab..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0011-test-add-latency-test-using-posixaio-ioengine.patch
+++ /dev/null
@@ -1,55 +0,0 @@
1From fa64b199318318af7fe598a5b9ec62b981a55e2d Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 15 Aug 2022 11:34:43 -0400
4Subject: [PATCH] test: add latency test using posixaio ioengine
5
6Make sure that mean(slat) + mean(clat) = mean(total lat).
7
8Tests 15 and 16 use the libaio and null ioengines, respectively. Both of
9those ioengines have commit hooks. Add this new test using the posixaio
10ioengine which does not have a commit hook so that we can better cover
11the possible ways that latency is calcualted.
12
13Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
14---
15 t/jobs/t0017.fio | 9 +++++++++
16 t/run-fio-tests.py | 10 ++++++++++
17 2 files changed, 19 insertions(+)
18 create mode 100644 t/jobs/t0017.fio
19
20diff --git a/t/jobs/t0017.fio b/t/jobs/t0017.fio
21new file mode 100644
22index 00000000..14486d98
23--- /dev/null
24+++ b/t/jobs/t0017.fio
25@@ -0,0 +1,9 @@
26+# Expected result: mean(slat) + mean(clat) = mean(lat)
27+# Buggy result: equality does not hold
28+# This is similar to t0015 and t0016 except that is uses posixaio which is
29+# available on more platforms and does not have a commit hook
30+
31+[test]
32+ioengine=posixaio
33+size=1M
34+iodepth=16
35diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
36index d77f20e0..2bd02a2a 100755
37--- a/t/run-fio-tests.py
38+++ b/t/run-fio-tests.py
39@@ -857,6 +857,16 @@ TEST_LIST = [
40 'output_format': 'json',
41 'requirements': [],
42 },
43+ {
44+ 'test_id': 17,
45+ 'test_class': FioJobTest_t0015,
46+ 'job': 't0017.fio',
47+ 'success': SUCCESS_DEFAULT,
48+ 'pre_job': None,
49+ 'pre_success': None,
50+ 'output_format': 'json',
51+ 'requirements': [Requirements.not_windows],
52+ },
53 {
54 'test_id': 1000,
55 'test_class': FioExeTest,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0012-test-fix-hash-for-t0016.patch b/meta-oe/recipes-benchmark/fio/fio/0012-test-fix-hash-for-t0016.patch
deleted file mode 100644
index 8f10f415c..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0012-test-fix-hash-for-t0016.patch
+++ /dev/null
@@ -1,33 +0,0 @@
1From 1e68459d85f56f805c70236ad47a65a65f426867 Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 15 Aug 2022 11:40:58 -0400
4Subject: [PATCH] test: fix hash for t0016
5
6I used the wrong hash for t0016 in the original commit. Fix it to refer
7to the hash that fixed the issue in this tree.
8
9Fixes: de31fe9a ("testing: add test for slat + clat = tlat")
10Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
11---
12 t/jobs/{t0016-259ebc00.fio => t0016-d54ae22.fio} | 0
13 t/run-fio-tests.py | 2 +-
14 2 files changed, 1 insertion(+), 1 deletion(-)
15 rename t/jobs/{t0016-259ebc00.fio => t0016-d54ae22.fio} (100%)
16
17diff --git a/t/jobs/t0016-259ebc00.fio b/t/jobs/t0016-d54ae22.fio
18similarity index 100%
19rename from t/jobs/t0016-259ebc00.fio
20rename to t/jobs/t0016-d54ae22.fio
21diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
22index 2bd02a2a..504b7cdb 100755
23--- a/t/run-fio-tests.py
24+++ b/t/run-fio-tests.py
25@@ -850,7 +850,7 @@ TEST_LIST = [
26 {
27 'test_id': 16,
28 'test_class': FioJobTest_t0015,
29- 'job': 't0016-259ebc00.fio',
30+ 'job': 't0016-d54ae22.fio',
31 'success': SUCCESS_DEFAULT,
32 'pre_job': None,
33 'pre_success': None,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0013-doc-get-rid-of-trailing-whitespace.patch b/meta-oe/recipes-benchmark/fio/fio/0013-doc-get-rid-of-trailing-whitespace.patch
deleted file mode 100644
index 5f72f47b0..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0013-doc-get-rid-of-trailing-whitespace.patch
+++ /dev/null
@@ -1,82 +0,0 @@
1From ff16b7c0c855d5242d92d7f902247525ff1f889a Mon Sep 17 00:00:00 2001
2From: Konstantin Kharlamov <Hi-Angel@yandex.ru>
3Date: Tue, 16 Aug 2022 19:14:13 +0300
4Subject: [PATCH] doc: get rid of trailing whitespace
5
6Signed-off-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
7---
8 HOWTO.rst | 4 ++--
9 fio.1 | 10 +++++-----
10 2 files changed, 7 insertions(+), 7 deletions(-)
11
12diff --git a/HOWTO.rst b/HOWTO.rst
13index b2750b56..c94238ed 100644
14--- a/HOWTO.rst
15+++ b/HOWTO.rst
16@@ -1301,7 +1301,7 @@ I/O type
17 effectively caps the file size at `real_size - offset`. Can be combined with
18 :option:`size` to constrain the start and end range of the I/O workload.
19 A percentage can be specified by a number between 1 and 100 followed by '%',
20- for example, ``offset=20%`` to specify 20%. In ZBD mode, value can be set as
21+ for example, ``offset=20%`` to specify 20%. In ZBD mode, value can be set as
22 number of zones using 'z'.
23
24 .. option:: offset_align=int
25@@ -1877,7 +1877,7 @@ I/O size
26 If this option is not specified, fio will use the full size of the given
27 files or devices. If the files do not exist, size must be given. It is also
28 possible to give size as a percentage between 1 and 100. If ``size=20%`` is
29- given, fio will use 20% of the full size of the given files or devices.
30+ given, fio will use 20% of the full size of the given files or devices.
31 In ZBD mode, value can also be set as number of zones using 'z'.
32 Can be combined with :option:`offset` to constrain the start and end range
33 that I/O will be done within.
34diff --git a/fio.1 b/fio.1
35index f3f3dc5d..d40b4247 100644
36--- a/fio.1
37+++ b/fio.1
38@@ -292,7 +292,7 @@ For Zone Block Device Mode:
39 .RS
40 .P
41 .PD 0
42-z means Zone
43+z means Zone
44 .P
45 .PD
46 .RE
47@@ -1083,7 +1083,7 @@ provided. Data before the given offset will not be touched. This
48 effectively caps the file size at `real_size \- offset'. Can be combined with
49 \fBsize\fR to constrain the start and end range of the I/O workload.
50 A percentage can be specified by a number between 1 and 100 followed by '%',
51-for example, `offset=20%' to specify 20%. In ZBD mode, value can be set as
52+for example, `offset=20%' to specify 20%. In ZBD mode, value can be set as
53 number of zones using 'z'.
54 .TP
55 .BI offset_align \fR=\fPint
56@@ -1099,7 +1099,7 @@ specified). This option is useful if there are several jobs which are
57 intended to operate on a file in parallel disjoint segments, with even
58 spacing between the starting points. Percentages can be used for this option.
59 If a percentage is given, the generated offset will be aligned to the minimum
60-\fBblocksize\fR or to the value of \fBoffset_align\fR if provided.In ZBD mode, value
61+\fBblocksize\fR or to the value of \fBoffset_align\fR if provided.In ZBD mode, value
62 can be set as number of zones using 'z'.
63 .TP
64 .BI number_ios \fR=\fPint
65@@ -1678,7 +1678,7 @@ If this option is not specified, fio will use the full size of the given
66 files or devices. If the files do not exist, size must be given. It is also
67 possible to give size as a percentage between 1 and 100. If `size=20%' is
68 given, fio will use 20% of the full size of the given files or devices. In ZBD mode,
69-size can be given in units of number of zones using 'z'. Can be combined with \fBoffset\fR to
70+size can be given in units of number of zones using 'z'. Can be combined with \fBoffset\fR to
71 constrain the start and end range that I/O will be done within.
72 .TP
73 .BI io_size \fR=\fPint[%|z] "\fR,\fB io_limit" \fR=\fPint[%|z]
74@@ -1697,7 +1697,7 @@ also be set as number of zones using 'z'.
75 .BI filesize \fR=\fPirange(int)
76 Individual file sizes. May be a range, in which case fio will select sizes
77 for files at random within the given range. If not given, each created file
78-is the same size. This option overrides \fBsize\fR in terms of file size,
79+is the same size. This option overrides \fBsize\fR in terms of file size,
80 i.e. \fBsize\fR becomes merely the default for \fBio_size\fR (and
81 has no effect it all if \fBio_size\fR is set explicitly).
82 .TP
diff --git a/meta-oe/recipes-benchmark/fio/fio/0014-doc-clarify-that-I-O-errors-may-go-unnoticed-without.patch b/meta-oe/recipes-benchmark/fio/fio/0014-doc-clarify-that-I-O-errors-may-go-unnoticed-without.patch
deleted file mode 100644
index 3196afa2b..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0014-doc-clarify-that-I-O-errors-may-go-unnoticed-without.patch
+++ /dev/null
@@ -1,54 +0,0 @@
1From 331023be2b20d177d533e5fa18f5d8834570613f Mon Sep 17 00:00:00 2001
2From: Konstantin Kharlamov <Hi-Angel@yandex.ru>
3Date: Tue, 16 Aug 2022 19:10:38 +0300
4Subject: [PATCH] doc: clarify that I/O errors may go unnoticed without
5 direct=1
6
7Fixes: https://github.com/axboe/fio/issues/1443
8
9Reported-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
10Signed-off-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
11---
12 HOWTO.rst | 7 +++++++
13 fio.1 | 10 ++++++++++
14 2 files changed, 17 insertions(+)
15
16diff --git a/HOWTO.rst b/HOWTO.rst
17index c94238ed..08be687c 100644
18--- a/HOWTO.rst
19+++ b/HOWTO.rst
20@@ -3927,6 +3927,13 @@ Error handling
21 appended, the total error count and the first error. The error field given
22 in the stats is the first error that was hit during the run.
23
24+ Note: a write error from the device may go unnoticed by fio when using
25+ buffered IO, as the write() (or similar) system call merely dirties the
26+ kernel pages, unless :option:`sync` or :option:`direct` is used. Device IO
27+ errors occur when the dirty data is actually written out to disk. If fully
28+ sync writes aren't desirable, :option:`fsync` or :option:`fdatasync` can be
29+ used as well. This is specific to writes, as reads are always synchronous.
30+
31 The allowed values are:
32
33 **none**
34diff --git a/fio.1 b/fio.1
35index d40b4247..27454b0b 100644
36--- a/fio.1
37+++ b/fio.1
38@@ -3606,6 +3606,16 @@ EILSEQ) until the runtime is exceeded or the I/O size specified is
39 completed. If this option is used, there are two more stats that are
40 appended, the total error count and the first error. The error field given
41 in the stats is the first error that was hit during the run.
42+.RS
43+.P
44+Note: a write error from the device may go unnoticed by fio when using buffered
45+IO, as the write() (or similar) system call merely dirties the kernel pages,
46+unless `sync' or `direct' is used. Device IO errors occur when the dirty data is
47+actually written out to disk. If fully sync writes aren't desirable, `fsync' or
48+`fdatasync' can be used as well. This is specific to writes, as reads are always
49+synchronous.
50+.RS
51+.P
52 The allowed values are:
53 .RS
54 .RS
diff --git a/meta-oe/recipes-benchmark/fio/fio/0015-Revert-Minor-style-fixups.patch b/meta-oe/recipes-benchmark/fio/fio/0015-Revert-Minor-style-fixups.patch
deleted file mode 100644
index fd962eaf2..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0015-Revert-Minor-style-fixups.patch
+++ /dev/null
@@ -1,41 +0,0 @@
1From 48ceba9c1870c9312d7214503371e0b781abba27 Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Tue, 23 Aug 2022 12:38:20 -0400
4Subject: [PATCH] Revert "Minor style fixups"
5
6This reverts commit 48f8268e88629d408ffd09b1601ad13366bd4ce1.
7
8Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
9---
10 backend.c | 2 +-
11 filesetup.c | 3 ++-
12 2 files changed, 3 insertions(+), 2 deletions(-)
13
14diff --git a/backend.c b/backend.c
15index 5159b60d..3a99850d 100644
16--- a/backend.c
17+++ b/backend.c
18@@ -2321,7 +2321,7 @@ static void run_threads(struct sk_out *sk_out)
19 * when setup_files() does not run into issues
20 * later.
21 */
22- if (!i && td->o.nr_files == 1) {
23+ if (!i && td->o.nr_files==1) {
24 if (setup_shared_file(td)) {
25 exit_value++;
26 if (td->error)
27diff --git a/filesetup.c b/filesetup.c
28index 3e2ccf9b..144a0572 100644
29--- a/filesetup.c
30+++ b/filesetup.c
31@@ -1109,8 +1109,9 @@ int setup_shared_file(struct thread_data *td)
32 dprint(FD_FILE, "fio: extending shared file\n");
33 f->real_file_size = file_size;
34 err = extend_file(td, f);
35- if (!err)
36+ if (!err) {
37 err = __file_invalidate_cache(td, f, 0, f->real_file_size);
38+ }
39 get_file_sizes(td);
40 dprint(FD_FILE, "shared setup new real_file_size=%llu\n",
41 (unsigned long long)f->real_file_size);
diff --git a/meta-oe/recipes-benchmark/fio/fio/0016-Revert-Fix-multithread-issues-when-operating-on-a-si.patch b/meta-oe/recipes-benchmark/fio/fio/0016-Revert-Fix-multithread-issues-when-operating-on-a-si.patch
deleted file mode 100644
index 31d834537..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0016-Revert-Fix-multithread-issues-when-operating-on-a-si.patch
+++ /dev/null
@@ -1,141 +0,0 @@
1From 1bc0dec3f54e67fa4767d0096ab377e900d5146f Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Tue, 23 Aug 2022 12:38:51 -0400
4Subject: [PATCH] Revert "Fix multithread issues when operating on a single
5 shared file"
6
7This reverts commit acbda87c34c743ff2d9e125d9539bcfbbf49eb75.
8
9This commit introduced a lot of unintended consequences for
10create_serialize=0. The aim of the commit can be accomplished with a
11combination of filesize and io_size.
12
13Fixes: https://github.com/axboe/fio/issues/1442
14Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
15---
16 backend.c | 19 +------------------
17 file.h | 1 -
18 filesetup.c | 46 ++--------------------------------------------
19 3 files changed, 3 insertions(+), 63 deletions(-)
20
21diff --git a/backend.c b/backend.c
22index 3a99850d..e5bb4e25 100644
23--- a/backend.c
24+++ b/backend.c
25@@ -2314,25 +2314,8 @@ static void run_threads(struct sk_out *sk_out)
26 for_each_td(td, i) {
27 print_status_init(td->thread_number - 1);
28
29- if (!td->o.create_serialize) {
30- /*
31- * When operating on a single rile in parallel,
32- * perform single-threaded early setup so that
33- * when setup_files() does not run into issues
34- * later.
35- */
36- if (!i && td->o.nr_files==1) {
37- if (setup_shared_file(td)) {
38- exit_value++;
39- if (td->error)
40- log_err("fio: pid=%d, err=%d/%s\n",
41- (int) td->pid, td->error, td->verror);
42- td_set_runstate(td, TD_REAPED);
43- todo--;
44- }
45- }
46+ if (!td->o.create_serialize)
47 continue;
48- }
49
50 if (fio_verify_load_state(td))
51 goto reap;
52diff --git a/file.h b/file.h
53index e646cf22..da1b8947 100644
54--- a/file.h
55+++ b/file.h
56@@ -201,7 +201,6 @@ struct thread_data;
57 extern void close_files(struct thread_data *);
58 extern void close_and_free_files(struct thread_data *);
59 extern uint64_t get_start_offset(struct thread_data *, struct fio_file *);
60-extern int __must_check setup_shared_file(struct thread_data *);
61 extern int __must_check setup_files(struct thread_data *);
62 extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *);
63 #ifdef __cplusplus
64diff --git a/filesetup.c b/filesetup.c
65index 144a0572..1d3cc5ad 100644
66--- a/filesetup.c
67+++ b/filesetup.c
68@@ -143,7 +143,7 @@ static int extend_file(struct thread_data *td, struct fio_file *f)
69 if (unlink_file || new_layout) {
70 int ret;
71
72- dprint(FD_FILE, "layout %d unlink %d %s\n", new_layout, unlink_file, f->file_name);
73+ dprint(FD_FILE, "layout unlink %s\n", f->file_name);
74
75 ret = td_io_unlink_file(td, f);
76 if (ret != 0 && ret != ENOENT) {
77@@ -198,9 +198,6 @@ static int extend_file(struct thread_data *td, struct fio_file *f)
78 }
79 }
80
81-
82- dprint(FD_FILE, "fill file %s, size %llu\n", f->file_name, (unsigned long long) f->real_file_size);
83-
84 left = f->real_file_size;
85 bs = td->o.max_bs[DDIR_WRITE];
86 if (bs > left)
87@@ -1081,45 +1078,6 @@ static bool create_work_dirs(struct thread_data *td, const char *fname)
88 return true;
89 }
90
91-int setup_shared_file(struct thread_data *td)
92-{
93- struct fio_file *f;
94- uint64_t file_size;
95- int err = 0;
96-
97- if (td->o.nr_files > 1) {
98- log_err("fio: shared file setup called for multiple files\n");
99- return -1;
100- }
101-
102- get_file_sizes(td);
103-
104- f = td->files[0];
105-
106- if (f == NULL) {
107- log_err("fio: NULL shared file\n");
108- return -1;
109- }
110-
111- file_size = thread_number * td->o.size;
112- dprint(FD_FILE, "shared setup %s real_file_size=%llu, desired=%llu\n",
113- f->file_name, (unsigned long long)f->real_file_size, (unsigned long long)file_size);
114-
115- if (f->real_file_size < file_size) {
116- dprint(FD_FILE, "fio: extending shared file\n");
117- f->real_file_size = file_size;
118- err = extend_file(td, f);
119- if (!err) {
120- err = __file_invalidate_cache(td, f, 0, f->real_file_size);
121- }
122- get_file_sizes(td);
123- dprint(FD_FILE, "shared setup new real_file_size=%llu\n",
124- (unsigned long long)f->real_file_size);
125- }
126-
127- return err;
128-}
129-
130 /*
131 * Open the files and setup files sizes, creating files if necessary.
132 */
133@@ -1134,7 +1092,7 @@ int setup_files(struct thread_data *td)
134 const unsigned long long bs = td_min_bs(td);
135 uint64_t fs = 0;
136
137- dprint(FD_FILE, "setup files (thread_number=%d, subjob_number=%d)\n", td->thread_number, td->subjob_number);
138+ dprint(FD_FILE, "setup files\n");
139
140 old_state = td_bump_runstate(td, TD_SETTING_UP);
141
diff --git a/meta-oe/recipes-benchmark/fio/fio/0017-Add-wait-for-handling-SIGBREAK.patch b/meta-oe/recipes-benchmark/fio/fio/0017-Add-wait-for-handling-SIGBREAK.patch
deleted file mode 100644
index e8cb33758..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0017-Add-wait-for-handling-SIGBREAK.patch
+++ /dev/null
@@ -1,59 +0,0 @@
1From 98beea79a30f1541e646efae911dfce10ae18f5c Mon Sep 17 00:00:00 2001
2From: Brandon Paupore <brandon.paupore@wdc.com>
3Date: Fri, 5 Aug 2022 12:57:27 -0500
4Subject: [PATCH] Add wait for handling SIGBREAK
5
6When closing a command prompt window or terminating it using something
7like the taskkill command, each child process (such as a running FIO
8workload) is sent a SIGBREAK signal. Once those child processes have
9responded to that signal, Windows terminates them if they're still
10executing.
11
12This change has the main thread to wait for others to exit when handling
13a SIGBREAK signal, such that each job will still have time to wrap-up
14and give stats before the entire program terminates.
15
16Signed-off-by: Brandon Paupore <brandon.paupore@wdc.com>
17---
18 backend.c | 21 ++++++++++++++++++++-
19 1 file changed, 20 insertions(+), 1 deletion(-)
20
21diff --git a/backend.c b/backend.c
22index e5bb4e25..375a23e4 100644
23--- a/backend.c
24+++ b/backend.c
25@@ -90,6 +90,25 @@ static void sig_int(int sig)
26 }
27 }
28
29+#ifdef WIN32
30+static void sig_break(int sig)
31+{
32+ struct thread_data *td;
33+ int i;
34+
35+ sig_int(sig);
36+
37+ /**
38+ * Windows terminates all job processes on SIGBREAK after the handler
39+ * returns, so give them time to wrap-up and give stats
40+ */
41+ for_each_td(td, i) {
42+ while (td->runstate < TD_EXITED)
43+ sleep(1);
44+ }
45+}
46+#endif
47+
48 void sig_show_status(int sig)
49 {
50 show_running_run_stats();
51@@ -112,7 +131,7 @@ static void set_sig_handlers(void)
52 /* Windows uses SIGBREAK as a quit signal from other applications */
53 #ifdef WIN32
54 memset(&act, 0, sizeof(act));
55- act.sa_handler = sig_int;
56+ act.sa_handler = sig_break;
57 act.sa_flags = SA_RESTART;
58 sigaction(SIGBREAK, &act, NULL);
59 #endif
diff --git a/meta-oe/recipes-benchmark/fio/fio/0018-engines-io_uring-pass-back-correct-error-value-when-.patch b/meta-oe/recipes-benchmark/fio/fio/0018-engines-io_uring-pass-back-correct-error-value-when-.patch
deleted file mode 100644
index 8aea3abac..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0018-engines-io_uring-pass-back-correct-error-value-when-.patch
+++ /dev/null
@@ -1,58 +0,0 @@
1From 1e6918419f4a2e5dbd77dd2da82598f1af63f533 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Wed, 24 Aug 2022 12:01:39 -0600
4Subject: [PATCH] engines/io_uring: pass back correct error value when
5 interrupted
6
7Running with an io_uring engine and using a USR1 signal to show
8current status will end up terminating the job with:
9
10fio: pid=91726, err=-4/file:ioengines.c:320, func=get_events, error=Unknown error -4
11sfx: (groupid=0, jobs=1): err=-4 (file:ioengines.c:320, func=get_events, error=Unknown error -4): pid=91726: Wed Aug 24 11:59:51 2022
12
13Ensure that the return value is set correctly based on the errno.
14
15Signed-off-by: Jens Axboe <axboe@kernel.dk>
16---
17 engines/io_uring.c | 8 ++++++++
18 1 file changed, 8 insertions(+)
19
20diff --git a/engines/io_uring.c b/engines/io_uring.c
21index cffc7371..89d64b06 100644
22--- a/engines/io_uring.c
23+++ b/engines/io_uring.c
24@@ -445,12 +445,18 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
25 struct io_uring_cqe *cqe;
26 struct io_u *io_u;
27 unsigned index;
28+ static int eio;
29
30 index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
31
32 cqe = &ld->cq_ring.cqes[index];
33 io_u = (struct io_u *) (uintptr_t) cqe->user_data;
34
35+ if (eio++ == 5) {
36+ printf("mark EIO\n");
37+ cqe->res = -EIO;
38+ }
39+
40 if (cqe->res != io_u->xfer_buflen) {
41 if (cqe->res > io_u->xfer_buflen)
42 io_u->error = -cqe->res;
43@@ -532,6 +538,7 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
44 if (r < 0) {
45 if (errno == EAGAIN || errno == EINTR)
46 continue;
47+ r = -errno;
48 td_verror(td, errno, "io_uring_enter");
49 break;
50 }
51@@ -665,6 +672,7 @@ static int fio_ioring_commit(struct thread_data *td)
52 usleep(1);
53 continue;
54 }
55+ ret = -errno;
56 td_verror(td, errno, "io_uring_enter submit");
57 break;
58 }
diff --git a/meta-oe/recipes-benchmark/fio/fio/0019-Enable-CPU-affinity-support-on-Android.patch b/meta-oe/recipes-benchmark/fio/fio/0019-Enable-CPU-affinity-support-on-Android.patch
deleted file mode 100644
index f47741612..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0019-Enable-CPU-affinity-support-on-Android.patch
+++ /dev/null
@@ -1,64 +0,0 @@
1From 02fd5b722bdfef2e7ce8d4aeb1bc65308d37003f Mon Sep 17 00:00:00 2001
2From: Bart Van Assche <bvanassche@acm.org>
3Date: Wed, 24 Aug 2022 13:42:29 -0700
4Subject: [PATCH] Enable CPU affinity support on Android
5
6This patch enables the --cpumask=, --cpus_allowed= and
7--cpus_allowed_policy= fio options.
8
9Signed-off-by: Bart Van Assche <bvanassche@acm.org>
10---
11 os/os-android.h | 26 ++++++++++++++++++++++++++
12 1 file changed, 26 insertions(+)
13
14diff --git a/os/os-android.h b/os/os-android.h
15index 2f73d249..34534239 100644
16--- a/os/os-android.h
17+++ b/os/os-android.h
18@@ -24,6 +24,7 @@
19 #define __has_builtin(x) 0 // Compatibility with non-clang compilers.
20 #endif
21
22+#define FIO_HAVE_CPU_AFFINITY
23 #define FIO_HAVE_DISK_UTIL
24 #define FIO_HAVE_IOSCHED_SWITCH
25 #define FIO_HAVE_IOPRIO
26@@ -44,6 +45,13 @@
27
28 #define OS_MAP_ANON MAP_ANONYMOUS
29
30+typedef cpu_set_t os_cpu_mask_t;
31+
32+#define fio_setaffinity(pid, cpumask) \
33+ sched_setaffinity((pid), sizeof(cpumask), &(cpumask))
34+#define fio_getaffinity(pid, ptr) \
35+ sched_getaffinity((pid), sizeof(cpu_set_t), (ptr))
36+
37 #ifndef POSIX_MADV_DONTNEED
38 #define posix_madvise madvise
39 #define POSIX_MADV_DONTNEED MADV_DONTNEED
40@@ -64,6 +72,24 @@
41 pthread_getaffinity_np(pthread_self(), sizeof(mask), &(mask))
42 #endif
43
44+#define fio_cpu_clear(mask, cpu) CPU_CLR((cpu), (mask))
45+#define fio_cpu_set(mask, cpu) CPU_SET((cpu), (mask))
46+#define fio_cpu_isset(mask, cpu) (CPU_ISSET((cpu), (mask)) != 0)
47+#define fio_cpu_count(mask) CPU_COUNT((mask))
48+
49+static inline int fio_cpuset_init(os_cpu_mask_t *mask)
50+{
51+ CPU_ZERO(mask);
52+ return 0;
53+}
54+
55+static inline int fio_cpuset_exit(os_cpu_mask_t *mask)
56+{
57+ return 0;
58+}
59+
60+#define FIO_MAX_CPUS CPU_SETSIZE
61+
62 #ifndef CONFIG_NO_SHM
63 /*
64 * Bionic doesn't support SysV shared memory, so implement it using ashmem
diff --git a/meta-oe/recipes-benchmark/fio/fio/0020-io_uring-Replace-pthread_self-with-s-tid.patch b/meta-oe/recipes-benchmark/fio/fio/0020-io_uring-Replace-pthread_self-with-s-tid.patch
deleted file mode 100644
index 24952f19f..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0020-io_uring-Replace-pthread_self-with-s-tid.patch
+++ /dev/null
@@ -1,41 +0,0 @@
1From 72d7f2139454528b9ebfb2f988a35f9a739680d0 Mon Sep 17 00:00:00 2001
2From: Khem Raj <raj.khem@gmail.com>
3Date: Wed, 24 Aug 2022 18:08:53 -0700
4Subject: [PATCH] io_uring: Replace pthread_self with s->tid
5
6__init_rand64 takes 64bit value and srand48 takes unsigned 32bit value,
7pthread_t is opaque type and some libcs ( e.g. musl ) do not define them
8in plain old data types and ends up with errors
9
10| t/io_uring.c:809:32: error: incompatible pointer to integer conversion passing 'pthread_t' (aka 'struct __pthread *') to parameter of type 'uint64_t' (aka 'unsigned long') [-Wint-conver
11sion]
12| __init_rand64(&s->rand_state, pthread_self());
13| ^~~~~~~~~~~~~~
14
15Signed-off-by: Khem Raj <raj.khem@gmail.com>
16---
17 t/io_uring.c | 5 ++---
18 1 file changed, 2 insertions(+), 3 deletions(-)
19
20diff --git a/t/io_uring.c b/t/io_uring.c
21index 35bf1956..f34a3554 100644
22--- a/t/io_uring.c
23+++ b/t/io_uring.c
24@@ -799,15 +799,14 @@ static int submitter_init(struct submitter *s)
25 int i, nr_batch, err;
26 static int init_printed;
27 char buf[80];
28-
29 s->tid = gettid();
30 printf("submitter=%d, tid=%d, file=%s, node=%d\n", s->index, s->tid,
31 s->filename, s->numa_node);
32
33 set_affinity(s);
34
35- __init_rand64(&s->rand_state, pthread_self());
36- srand48(pthread_self());
37+ __init_rand64(&s->rand_state, s->tid);
38+ srand48(s->tid);
39
40 for (i = 0; i < MAX_FDS; i++)
41 s->files[i].fileno = i;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0021-engines-io_uring-delete-debug-code.patch b/meta-oe/recipes-benchmark/fio/fio/0021-engines-io_uring-delete-debug-code.patch
deleted file mode 100644
index b4f3d0d27..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0021-engines-io_uring-delete-debug-code.patch
+++ /dev/null
@@ -1,37 +0,0 @@
1From 85f8181d42050f8a8c9ddf6d30f621054f0e6890 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Thu, 25 Aug 2022 11:19:34 -0600
4Subject: [PATCH] engines/io_uring: delete debug code
5
6This was inadvertently introduced by a previous commit, get rid
7of it.
8
9Fixes: 1816895b788e ("engines/io_uring: pass back correct error value when interrupted")
10Signed-off-by: Jens Axboe <axboe@kernel.dk>
11---
12 engines/io_uring.c | 6 ------
13 1 file changed, 6 deletions(-)
14
15diff --git a/engines/io_uring.c b/engines/io_uring.c
16index 89d64b06..94376efa 100644
17--- a/engines/io_uring.c
18+++ b/engines/io_uring.c
19@@ -445,18 +445,12 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
20 struct io_uring_cqe *cqe;
21 struct io_u *io_u;
22 unsigned index;
23- static int eio;
24
25 index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
26
27 cqe = &ld->cq_ring.cqes[index];
28 io_u = (struct io_u *) (uintptr_t) cqe->user_data;
29
30- if (eio++ == 5) {
31- printf("mark EIO\n");
32- cqe->res = -EIO;
33- }
34-
35 if (cqe->res != io_u->xfer_buflen) {
36 if (cqe->res > io_u->xfer_buflen)
37 io_u->error = -cqe->res;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0022-t-io_uring-prep-for-including-engines-nvme.h-in-t-io.patch b/meta-oe/recipes-benchmark/fio/fio/0022-t-io_uring-prep-for-including-engines-nvme.h-in-t-io.patch
deleted file mode 100644
index f32c8867d..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0022-t-io_uring-prep-for-including-engines-nvme.h-in-t-io.patch
+++ /dev/null
@@ -1,72 +0,0 @@
1From a10cec0440a7574ffb76ff52fbc33a250f067d6a Mon Sep 17 00:00:00 2001
2From: Anuj Gupta <anuj20.g@samsung.com>
3Date: Fri, 26 Aug 2022 17:03:05 +0530
4Subject: [PATCH] t/io_uring: prep for including engines/nvme.h in t/io_uring
5
6Change page_size and cal_clat_percentiles name to something different
7as these are indirectly picked from engines/nvme.h (fio.h and stat.h)
8
9Signed-off-by: Anuj Gupta <anuj20.g@samsung.com>
10Link: https://lore.kernel.org/r/20220826113306.4139-2-anuj20.g@samsung.com
11Signed-off-by: Jens Axboe <axboe@kernel.dk>
12---
13 t/io_uring.c | 18 +++++++++---------
14 1 file changed, 9 insertions(+), 9 deletions(-)
15
16diff --git a/t/io_uring.c b/t/io_uring.c
17index f34a3554..6e4737e4 100644
18--- a/t/io_uring.c
19+++ b/t/io_uring.c
20@@ -117,7 +117,7 @@ static struct submitter *submitter;
21 static volatile int finish;
22 static int stats_running;
23 static unsigned long max_iops;
24-static long page_size;
25+static long t_io_uring_page_size;
26
27 static int depth = DEPTH;
28 static int batch_submit = BATCH_SUBMIT;
29@@ -195,9 +195,9 @@ static unsigned long plat_idx_to_val(unsigned int idx)
30 return cycles_to_nsec(base + ((k + 0.5) * (1 << error_bits)));
31 }
32
33-unsigned int calc_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
34- unsigned long **output,
35- unsigned long *maxv, unsigned long *minv)
36+unsigned int calculate_clat_percentiles(unsigned long *io_u_plat,
37+ unsigned long nr, unsigned long **output,
38+ unsigned long *maxv, unsigned long *minv)
39 {
40 unsigned long sum = 0;
41 unsigned int len = plist_len, i, j = 0;
42@@ -251,7 +251,7 @@ static void show_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
43 bool is_last;
44 char fmt[32];
45
46- len = calc_clat_percentiles(io_u_plat, nr, &ovals, &maxv, &minv);
47+ len = calculate_clat_percentiles(io_u_plat, nr, &ovals, &maxv, &minv);
48 if (!len || !ovals)
49 goto out;
50
51@@ -786,7 +786,7 @@ static void *allocate_mem(struct submitter *s, int size)
52 return numa_alloc_onnode(size, s->numa_node);
53 #endif
54
55- if (posix_memalign(&buf, page_size, bs)) {
56+ if (posix_memalign(&buf, t_io_uring_page_size, bs)) {
57 printf("failed alloc\n");
58 return NULL;
59 }
60@@ -1542,9 +1542,9 @@ int main(int argc, char *argv[])
61
62 arm_sig_int();
63
64- page_size = sysconf(_SC_PAGESIZE);
65- if (page_size < 0)
66- page_size = 4096;
67+ t_io_uring_page_size = sysconf(_SC_PAGESIZE);
68+ if (t_io_uring_page_size < 0)
69+ t_io_uring_page_size = 4096;
70
71 for (j = 0; j < nthreads; j++) {
72 s = get_submitter(j);
diff --git a/meta-oe/recipes-benchmark/fio/fio/0023-t-io_uring-add-support-for-async-passthru.patch b/meta-oe/recipes-benchmark/fio/fio/0023-t-io_uring-add-support-for-async-passthru.patch
deleted file mode 100644
index 8ef7d13e9..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0023-t-io_uring-add-support-for-async-passthru.patch
+++ /dev/null
@@ -1,379 +0,0 @@
1From fa4a1345ca7cd60ae0b96da286f45621a3f45a33 Mon Sep 17 00:00:00 2001
2From: Anuj Gupta <anuj20.g@samsung.com>
3Date: Fri, 26 Aug 2022 17:03:06 +0530
4Subject: [PATCH] t/io_uring: add support for async-passthru
5
6This patch adds support for async-passthru in t/io_uring. User needs to
7specify -u1 option in the command
8
9Example commandline:
10t/io_uring -b512 -d128 -c32 -s32 -p0 -F1 -B0 -O0 -n1 -u1 /dev/ng0n1
11
12Signed-off-by: Anuj Gupta <anuj20.g@samsung.com>
13Link: https://lore.kernel.org/r/20220826113306.4139-3-anuj20.g@samsung.com
14Signed-off-by: Jens Axboe <axboe@kernel.dk>
15---
16 t/io_uring.c | 238 +++++++++++++++++++++++++++++++++++++++++++++++++--
17 1 file changed, 230 insertions(+), 8 deletions(-)
18
19diff --git a/t/io_uring.c b/t/io_uring.c
20index 6e4737e4..0a90f85c 100644
21--- a/t/io_uring.c
22+++ b/t/io_uring.c
23@@ -35,6 +35,7 @@
24 #include "../lib/rand.h"
25 #include "../minmax.h"
26 #include "../os/linux/io_uring.h"
27+#include "../engines/nvme.h"
28
29 struct io_sq_ring {
30 unsigned *head;
31@@ -67,6 +68,8 @@ struct file {
32 unsigned long max_size;
33 unsigned long cur_off;
34 unsigned pending_ios;
35+ unsigned int nsid; /* nsid field required for nvme-passthrough */
36+ unsigned int lba_shift; /* lba_shift field required for nvme-passthrough */
37 int real_fd;
38 int fixed_fd;
39 int fileno;
40@@ -139,6 +142,7 @@ static int random_io = 1; /* random or sequential IO */
41 static int register_ring = 1; /* register ring */
42 static int use_sync = 0; /* use preadv2 */
43 static int numa_placement = 0; /* set to node of device */
44+static int pt = 0; /* passthrough I/O or not */
45
46 static unsigned long tsc_rate;
47
48@@ -161,6 +165,54 @@ struct io_uring_map_buffers {
49 };
50 #endif
51
52+static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
53+ enum nvme_csi csi, void *data)
54+{
55+ struct nvme_passthru_cmd cmd = {
56+ .opcode = nvme_admin_identify,
57+ .nsid = nsid,
58+ .addr = (__u64)(uintptr_t)data,
59+ .data_len = NVME_IDENTIFY_DATA_SIZE,
60+ .cdw10 = cns,
61+ .cdw11 = csi << NVME_IDENTIFY_CSI_SHIFT,
62+ .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
63+ };
64+
65+ return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
66+}
67+
68+static int nvme_get_info(int fd, __u32 *nsid, __u32 *lba_sz, __u64 *nlba)
69+{
70+ struct nvme_id_ns ns;
71+ int namespace_id;
72+ int err;
73+
74+ namespace_id = ioctl(fd, NVME_IOCTL_ID);
75+ if (namespace_id < 0) {
76+ fprintf(stderr, "error failed to fetch namespace-id\n");
77+ close(fd);
78+ return -errno;
79+ }
80+
81+ /*
82+ * Identify namespace to get namespace-id, namespace size in LBA's
83+ * and LBA data size.
84+ */
85+ err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_NS,
86+ NVME_CSI_NVM, &ns);
87+ if (err) {
88+ fprintf(stderr, "error failed to fetch identify namespace\n");
89+ close(fd);
90+ return err;
91+ }
92+
93+ *nsid = namespace_id;
94+ *lba_sz = 1 << ns.lbaf[(ns.flbas & 0x0f)].ds;
95+ *nlba = ns.nsze;
96+
97+ return 0;
98+}
99+
100 static unsigned long cycles_to_nsec(unsigned long cycles)
101 {
102 uint64_t val;
103@@ -520,6 +572,65 @@ static void init_io(struct submitter *s, unsigned index)
104 sqe->user_data |= ((uint64_t)s->clock_index << 32);
105 }
106
107+static void init_io_pt(struct submitter *s, unsigned index)
108+{
109+ struct io_uring_sqe *sqe = &s->sqes[index << 1];
110+ unsigned long offset;
111+ struct file *f;
112+ struct nvme_uring_cmd *cmd;
113+ unsigned long long slba;
114+ unsigned long long nlb;
115+ long r;
116+
117+ if (s->nr_files == 1) {
118+ f = &s->files[0];
119+ } else {
120+ f = &s->files[s->cur_file];
121+ if (f->pending_ios >= file_depth(s)) {
122+ s->cur_file++;
123+ if (s->cur_file == s->nr_files)
124+ s->cur_file = 0;
125+ f = &s->files[s->cur_file];
126+ }
127+ }
128+ f->pending_ios++;
129+
130+ if (random_io) {
131+ r = __rand64(&s->rand_state);
132+ offset = (r % (f->max_blocks - 1)) * bs;
133+ } else {
134+ offset = f->cur_off;
135+ f->cur_off += bs;
136+ if (f->cur_off + bs > f->max_size)
137+ f->cur_off = 0;
138+ }
139+
140+ if (register_files) {
141+ sqe->fd = f->fixed_fd;
142+ sqe->flags = IOSQE_FIXED_FILE;
143+ } else {
144+ sqe->fd = f->real_fd;
145+ sqe->flags = 0;
146+ }
147+ sqe->opcode = IORING_OP_URING_CMD;
148+ sqe->user_data = (unsigned long) f->fileno;
149+ if (stats)
150+ sqe->user_data |= ((unsigned long)s->clock_index << 32);
151+ sqe->cmd_op = NVME_URING_CMD_IO;
152+ slba = offset >> f->lba_shift;
153+ nlb = (bs >> f->lba_shift) - 1;
154+ cmd = (struct nvme_uring_cmd *)&sqe->cmd;
155+ /* cdw10 and cdw11 represent starting slba*/
156+ cmd->cdw10 = slba & 0xffffffff;
157+ cmd->cdw11 = slba >> 32;
158+ /* cdw12 represent number of lba to be read*/
159+ cmd->cdw12 = nlb;
160+ cmd->addr = (unsigned long) s->iovecs[index].iov_base;
161+ cmd->data_len = bs;
162+ cmd->nsid = f->nsid;
163+ cmd->opcode = 2;
164+}
165+
166 static int prep_more_ios_uring(struct submitter *s, int max_ios)
167 {
168 struct io_sq_ring *ring = &s->sq_ring;
169@@ -532,7 +643,10 @@ static int prep_more_ios_uring(struct submitter *s, int max_ios)
170 break;
171
172 index = tail & sq_ring_mask;
173- init_io(s, index);
174+ if (pt)
175+ init_io_pt(s, index);
176+ else
177+ init_io(s, index);
178 ring->array[index] = index;
179 prepped++;
180 tail = next_tail;
181@@ -549,7 +663,29 @@ static int get_file_size(struct file *f)
182
183 if (fstat(f->real_fd, &st) < 0)
184 return -1;
185- if (S_ISBLK(st.st_mode)) {
186+ if (pt) {
187+ __u64 nlba;
188+ __u32 lbs;
189+ int ret;
190+
191+ if (!S_ISCHR(st.st_mode)) {
192+ fprintf(stderr, "passthrough works with only nvme-ns "
193+ "generic devices (/dev/ngXnY)\n");
194+ return -1;
195+ }
196+ ret = nvme_get_info(f->real_fd, &f->nsid, &lbs, &nlba);
197+ if (ret)
198+ return -1;
199+ if ((bs % lbs) != 0) {
200+ printf("error: bs:%d should be a multiple logical_block_size:%d\n",
201+ bs, lbs);
202+ return -1;
203+ }
204+ f->max_blocks = nlba / bs;
205+ f->max_size = nlba;
206+ f->lba_shift = ilog2(lbs);
207+ return 0;
208+ } else if (S_ISBLK(st.st_mode)) {
209 unsigned long long bytes;
210
211 if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
212@@ -620,6 +756,60 @@ static int reap_events_uring(struct submitter *s)
213 return reaped;
214 }
215
216+static int reap_events_uring_pt(struct submitter *s)
217+{
218+ struct io_cq_ring *ring = &s->cq_ring;
219+ struct io_uring_cqe *cqe;
220+ unsigned head, reaped = 0;
221+ int last_idx = -1, stat_nr = 0;
222+ unsigned index;
223+ int fileno;
224+
225+ head = *ring->head;
226+ do {
227+ struct file *f;
228+
229+ read_barrier();
230+ if (head == atomic_load_acquire(ring->tail))
231+ break;
232+ index = head & cq_ring_mask;
233+ cqe = &ring->cqes[index << 1];
234+ fileno = cqe->user_data & 0xffffffff;
235+ f = &s->files[fileno];
236+ f->pending_ios--;
237+
238+ if (cqe->res != 0) {
239+ printf("io: unexpected ret=%d\n", cqe->res);
240+ if (polled && cqe->res == -EINVAL)
241+ printf("passthrough doesn't support polled IO\n");
242+ return -1;
243+ }
244+ if (stats) {
245+ int clock_index = cqe->user_data >> 32;
246+
247+ if (last_idx != clock_index) {
248+ if (last_idx != -1) {
249+ add_stat(s, last_idx, stat_nr);
250+ stat_nr = 0;
251+ }
252+ last_idx = clock_index;
253+ }
254+ stat_nr++;
255+ }
256+ reaped++;
257+ head++;
258+ } while (1);
259+
260+ if (stat_nr)
261+ add_stat(s, last_idx, stat_nr);
262+
263+ if (reaped) {
264+ s->inflight -= reaped;
265+ atomic_store_release(ring->head, head);
266+ }
267+ return reaped;
268+}
269+
270 static void set_affinity(struct submitter *s)
271 {
272 #ifdef CONFIG_LIBNUMA
273@@ -697,6 +887,7 @@ static int setup_ring(struct submitter *s)
274 struct io_uring_params p;
275 int ret, fd;
276 void *ptr;
277+ size_t len;
278
279 memset(&p, 0, sizeof(p));
280
281@@ -709,6 +900,10 @@ static int setup_ring(struct submitter *s)
282 p.sq_thread_cpu = sq_thread_cpu;
283 }
284 }
285+ if (pt) {
286+ p.flags |= IORING_SETUP_SQE128;
287+ p.flags |= IORING_SETUP_CQE32;
288+ }
289
290 fd = io_uring_setup(depth, &p);
291 if (fd < 0) {
292@@ -761,11 +956,22 @@ static int setup_ring(struct submitter *s)
293 sring->array = ptr + p.sq_off.array;
294 sq_ring_mask = *sring->ring_mask;
295
296- s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
297+ if (p.flags & IORING_SETUP_SQE128)
298+ len = 2 * p.sq_entries * sizeof(struct io_uring_sqe);
299+ else
300+ len = p.sq_entries * sizeof(struct io_uring_sqe);
301+ s->sqes = mmap(0, len,
302 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
303 IORING_OFF_SQES);
304
305- ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
306+ if (p.flags & IORING_SETUP_CQE32) {
307+ len = p.cq_off.cqes +
308+ 2 * p.cq_entries * sizeof(struct io_uring_cqe);
309+ } else {
310+ len = p.cq_off.cqes +
311+ p.cq_entries * sizeof(struct io_uring_cqe);
312+ }
313+ ptr = mmap(0, len,
314 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
315 IORING_OFF_CQ_RING);
316 cring->head = ptr + p.cq_off.head;
317@@ -855,7 +1061,16 @@ static int submitter_init(struct submitter *s)
318 s->plat = NULL;
319 nr_batch = 0;
320 }
321+ /* perform the expensive command initialization part for passthrough here
322+ * rather than in the fast path
323+ */
324+ if (pt) {
325+ for (i = 0; i < roundup_pow2(depth); i++) {
326+ struct io_uring_sqe *sqe = &s->sqes[i << 1];
327
328+ memset(&sqe->cmd, 0, sizeof(struct nvme_uring_cmd));
329+ }
330+ }
331 return nr_batch;
332 }
333
334@@ -1111,7 +1326,10 @@ submit:
335 do {
336 int r;
337
338- r = reap_events_uring(s);
339+ if (pt)
340+ r = reap_events_uring_pt(s);
341+ else
342+ r = reap_events_uring(s);
343 if (r == -1) {
344 s->finish = 1;
345 break;
346@@ -1305,11 +1523,12 @@ static void usage(char *argv, int status)
347 " -a <bool> : Use legacy aio, default %d\n"
348 " -S <bool> : Use sync IO (preadv2), default %d\n"
349 " -X <bool> : Use registered ring %d\n"
350- " -P <bool> : Automatically place on device home node %d\n",
351+ " -P <bool> : Automatically place on device home node %d\n"
352+ " -u <bool> : Use nvme-passthrough I/O, default %d\n",
353 argv, DEPTH, BATCH_SUBMIT, BATCH_COMPLETE, BS, polled,
354 fixedbufs, dma_map, register_files, nthreads, !buffered, do_nop,
355 stats, runtime == 0 ? "unlimited" : runtime_str, random_io, aio,
356- use_sync, register_ring, numa_placement);
357+ use_sync, register_ring, numa_placement, pt);
358 exit(status);
359 }
360
361@@ -1368,7 +1587,7 @@ int main(int argc, char *argv[])
362 if (!do_nop && argc < 2)
363 usage(argv[0], 1);
364
365- while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:a:r:D:R:X:S:P:h?")) != -1) {
366+ while ((opt = getopt(argc, argv, "d:s:c:b:p:B:F:n:N:O:t:T:a:r:D:R:X:S:P:u:h?")) != -1) {
367 switch (opt) {
368 case 'a':
369 aio = !!atoi(optarg);
370@@ -1449,6 +1668,9 @@ int main(int argc, char *argv[])
371 case 'P':
372 numa_placement = !!atoi(optarg);
373 break;
374+ case 'u':
375+ pt = !!atoi(optarg);
376+ break;
377 case 'h':
378 case '?':
379 default:
diff --git a/meta-oe/recipes-benchmark/fio/fio/0024-t-io_uring-fix-64-bit-cast-on-32-bit-archs.patch b/meta-oe/recipes-benchmark/fio/fio/0024-t-io_uring-fix-64-bit-cast-on-32-bit-archs.patch
deleted file mode 100644
index ba687e041..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0024-t-io_uring-fix-64-bit-cast-on-32-bit-archs.patch
+++ /dev/null
@@ -1,37 +0,0 @@
1From 286bed8bc95fbc7d8a1d00b1861037bc215948ee Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Fri, 26 Aug 2022 07:52:54 -0600
4Subject: [PATCH] t/io_uring: fix 64-bit cast on 32-bit archs
5MIME-Version: 1.0
6Content-Type: text/plain; charset=UTF-8
7Content-Transfer-Encoding: 8bit
8
9gcc complains that:
10
11t/io_uring.c: In function ‘init_io_pt’:
12t/io_uring.c:618:52: error: left shift count >= width of type [-Werror=shift-count-overflow]
13 618 | sqe->user_data |= ((unsigned long)s->clock_index << 32);
14 | ^~
15
16we're shifting more than the size of the type. Cast to a 64-bit value
17so that it'll work on 32-bit as well.
18
19Fixes: 7d04588a7663 ("t/io_uring: add support for async-passthru")
20Signed-off-by: Jens Axboe <axboe@kernel.dk>
21---
22 t/io_uring.c | 2 +-
23 1 file changed, 1 insertion(+), 1 deletion(-)
24
25diff --git a/t/io_uring.c b/t/io_uring.c
26index 0a90f85c..b90bcf78 100644
27--- a/t/io_uring.c
28+++ b/t/io_uring.c
29@@ -615,7 +615,7 @@ static void init_io_pt(struct submitter *s, unsigned index)
30 sqe->opcode = IORING_OP_URING_CMD;
31 sqe->user_data = (unsigned long) f->fileno;
32 if (stats)
33- sqe->user_data |= ((unsigned long)s->clock_index << 32);
34+ sqe->user_data |= ((__u64) s->clock_index << 32ULL);
35 sqe->cmd_op = NVME_URING_CMD_IO;
36 slba = offset >> f->lba_shift;
37 nlb = (bs >> f->lba_shift) - 1;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0025-test-add-basic-test-for-io_uring-ioengine.patch b/meta-oe/recipes-benchmark/fio/fio/0025-test-add-basic-test-for-io_uring-ioengine.patch
deleted file mode 100644
index 7472eea62..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0025-test-add-basic-test-for-io_uring-ioengine.patch
+++ /dev/null
@@ -1,91 +0,0 @@
1From c3df3532a507e6d7c66339dee6eb022feab68f59 Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Thu, 25 Aug 2022 12:08:33 -0700
4Subject: [PATCH] test: add basic test for io_uring ioengine
5
6We should have a quick smoke test for the io_uring ioengine to
7automatically detect breakage.
8
9Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
10---
11 t/jobs/t0018.fio | 9 +++++++++
12 t/run-fio-tests.py | 22 ++++++++++++++++++++++
13 2 files changed, 31 insertions(+)
14 create mode 100644 t/jobs/t0018.fio
15
16diff --git a/t/jobs/t0018.fio b/t/jobs/t0018.fio
17new file mode 100644
18index 00000000..e2298b1f
19--- /dev/null
20+++ b/t/jobs/t0018.fio
21@@ -0,0 +1,9 @@
22+# Expected result: job completes without error
23+# Buggy result: job fails
24+
25+[test]
26+ioengine=io_uring
27+filesize=256K
28+time_based
29+runtime=3s
30+rw=randrw
31diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
32index 504b7cdb..1e5e9f24 100755
33--- a/t/run-fio-tests.py
34+++ b/t/run-fio-tests.py
35@@ -582,6 +582,7 @@ class Requirements(object):
36
37 _linux = False
38 _libaio = False
39+ _io_uring = False
40 _zbd = False
41 _root = False
42 _zoned_nullb = False
43@@ -605,6 +606,12 @@ class Requirements(object):
44 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
45 Requirements._libaio = "CONFIG_LIBAIO" in contents
46
47+ contents, success = FioJobTest.get_file("/proc/kallsyms")
48+ if not success:
49+ print("Unable to open '/proc/kallsyms' to probe for io_uring support")
50+ else:
51+ Requirements._io_uring = "io_uring_setup" in contents
52+
53 Requirements._root = (os.geteuid() == 0)
54 if Requirements._zbd and Requirements._root:
55 try:
56@@ -627,6 +634,7 @@ class Requirements(object):
57
58 req_list = [Requirements.linux,
59 Requirements.libaio,
60+ Requirements.io_uring,
61 Requirements.zbd,
62 Requirements.root,
63 Requirements.zoned_nullb,
64@@ -648,6 +656,11 @@ class Requirements(object):
65 """Is libaio available?"""
66 return Requirements._libaio, "libaio required"
67
68+ @classmethod
69+ def io_uring(cls):
70+ """Is io_uring available?"""
71+ return Requirements._io_uring, "io_uring required"
72+
73 @classmethod
74 def zbd(cls):
75 """Is ZBD support available?"""
76@@ -867,6 +880,15 @@ TEST_LIST = [
77 'output_format': 'json',
78 'requirements': [Requirements.not_windows],
79 },
80+ {
81+ 'test_id': 18,
82+ 'test_class': FioJobTest,
83+ 'job': 't0018.fio',
84+ 'success': SUCCESS_DEFAULT,
85+ 'pre_job': None,
86+ 'pre_success': None,
87+ 'requirements': [Requirements.linux, Requirements.io_uring],
88+ },
89 {
90 'test_id': 1000,
91 'test_class': FioExeTest,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0026-t-io_uring-remove-duplicate-definition-of-gettid.patch b/meta-oe/recipes-benchmark/fio/fio/0026-t-io_uring-remove-duplicate-definition-of-gettid.patch
deleted file mode 100644
index c75ec36d1..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0026-t-io_uring-remove-duplicate-definition-of-gettid.patch
+++ /dev/null
@@ -1,59 +0,0 @@
1From 575bea1c9b642a11ac5b7162aea6a9f905c60318 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Fri, 26 Aug 2022 14:14:44 -0600
4Subject: [PATCH] t/io_uring: remove duplicate definition of gettid()
5MIME-Version: 1.0
6Content-Type: text/plain; charset=UTF-8
7Content-Transfer-Encoding: 8bit
8
9With a recent change, we now include os.h through nvme.h, and this
10can cause a duplicate gettid() definition:
11
12t/io_uring.c:499:12: error: redefinition of ‘gettid’
13 static int gettid(void)
14 ^~~~~~
15In file included from t/../engines/../os/os.h:39,
16 from t/../engines/../thread_options.h:5,
17 from t/../engines/../fio.h:18,
18 from t/../engines/nvme.h:10,
19 from t/io_uring.c:38:
20t/../engines/../os/os-linux.h:147:19: note: previous definition of
21‘gettid’ was here
22 static inline int gettid(void)
23 ^~~~~~
24
25Include os.h directly to make it clear that we use it, and remove the
26gettid() definition from io_uring.c.
27
28Reported-by: Yi Zhang <yi.zhang@redhat.com>
29Signed-off-by: Jens Axboe <axboe@kernel.dk>
30---
31 t/io_uring.c | 8 +-------
32 1 file changed, 1 insertion(+), 7 deletions(-)
33
34diff --git a/t/io_uring.c b/t/io_uring.c
35index b90bcf78..e8e41796 100644
36--- a/t/io_uring.c
37+++ b/t/io_uring.c
38@@ -30,6 +30,7 @@
39 #include <sched.h>
40
41 #include "../arch/arch.h"
42+#include "../os/os.h"
43 #include "../lib/types.h"
44 #include "../lib/roundup.h"
45 #include "../lib/rand.h"
46@@ -495,13 +496,6 @@ static int io_uring_enter(struct submitter *s, unsigned int to_submit,
47 #endif
48 }
49
50-#ifndef CONFIG_HAVE_GETTID
51-static int gettid(void)
52-{
53- return syscall(__NR_gettid);
54-}
55-#endif
56-
57 static unsigned file_depth(struct submitter *s)
58 {
59 return (depth + s->nr_files - 1) / s->nr_files;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0027-test-add-some-tests-for-seq-and-rand-offsets.patch b/meta-oe/recipes-benchmark/fio/fio/0027-test-add-some-tests-for-seq-and-rand-offsets.patch
deleted file mode 100644
index 2abd449b8..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0027-test-add-some-tests-for-seq-and-rand-offsets.patch
+++ /dev/null
@@ -1,157 +0,0 @@
1From 1eef6cddda678b0d1a120970bc4cc961c285c81e Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 29 Aug 2022 11:30:30 -0400
4Subject: [PATCH] test: add some tests for seq and rand offsets
5
6t/jobs/t0019.fio is a seq read test
7t/jobs/t0020.fio is a rand read test
8
9We don't have any automated tests which make sure that sequential access
10patterns are actually sequential and that random access patterns are not
11sequential. Add these two tests to help detect the possibility that
12these features could break.
13
14Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
15---
16 t/jobs/t0019.fio | 10 ++++++
17 t/jobs/t0020.fio | 11 ++++++
18 t/run-fio-tests.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++
19 3 files changed, 105 insertions(+)
20 create mode 100644 t/jobs/t0019.fio
21 create mode 100644 t/jobs/t0020.fio
22
23diff --git a/t/jobs/t0019.fio b/t/jobs/t0019.fio
24new file mode 100644
25index 00000000..b60d27d2
26--- /dev/null
27+++ b/t/jobs/t0019.fio
28@@ -0,0 +1,10 @@
29+# Expected result: offsets are accessed sequentially and all offsets are read
30+# Buggy result: offsets are not accessed sequentially and one or more offsets are missed
31+# run with --debug=io or logging to see which offsets are accessed
32+
33+[test]
34+ioengine=null
35+filesize=1M
36+write_bw_log=test
37+per_job_logs=0
38+log_offset=1
39diff --git a/t/jobs/t0020.fio b/t/jobs/t0020.fio
40new file mode 100644
41index 00000000..1c1c5166
42--- /dev/null
43+++ b/t/jobs/t0020.fio
44@@ -0,0 +1,11 @@
45+# Expected result: offsets are not accessed sequentially and all offsets are touched
46+# Buggy result: offsets are accessed sequentially and one or more offsets are missed
47+# run with --debug=io or logging to see which offsets are read
48+
49+[test]
50+ioengine=null
51+filesize=1M
52+rw=randread
53+write_bw_log=test
54+per_job_logs=0
55+log_offset=1
56diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
57index 1e5e9f24..78f43521 100755
58--- a/t/run-fio-tests.py
59+++ b/t/run-fio-tests.py
60@@ -548,6 +548,72 @@ class FioJobTest_t0015(FioJobTest):
61 self.passed = False
62
63
64+class FioJobTest_t0019(FioJobTest):
65+ """Test consists of fio test job t0019
66+ Confirm that all offsets were touched sequentially"""
67+
68+ def check_result(self):
69+ super(FioJobTest_t0019, self).check_result()
70+
71+ bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
72+ file_data, success = self.get_file(bw_log_filename)
73+ log_lines = file_data.split('\n')
74+
75+ prev = -4096
76+ for line in log_lines:
77+ if len(line.strip()) == 0:
78+ continue
79+ cur = int(line.split(',')[4])
80+ if cur - prev != 4096:
81+ self.passed = False
82+ self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur)
83+ return
84+ prev = cur
85+
86+ if cur/4096 != 255:
87+ self.passed = False
88+ self.failure_reason = "unexpected last offset {0}".format(cur)
89+
90+
91+class FioJobTest_t0020(FioJobTest):
92+ """Test consists of fio test job t0020
93+ Confirm that almost all offsets were touched non-sequentially"""
94+
95+ def check_result(self):
96+ super(FioJobTest_t0020, self).check_result()
97+
98+ bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
99+ file_data, success = self.get_file(bw_log_filename)
100+ log_lines = file_data.split('\n')
101+
102+ seq_count = 0
103+ offsets = set()
104+
105+ prev = int(log_lines[0].split(',')[4])
106+ for line in log_lines[1:]:
107+ offsets.add(prev/4096)
108+ if len(line.strip()) == 0:
109+ continue
110+ cur = int(line.split(',')[4])
111+ if cur - prev == 4096:
112+ seq_count += 1
113+ prev = cur
114+
115+ # 10 is an arbitrary threshold
116+ if seq_count > 10:
117+ self.passed = False
118+ self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
119+
120+ if len(offsets) != 256:
121+ self.passed = False
122+ self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
123+
124+ for i in range(256):
125+ if not i in offsets:
126+ self.passed = False
127+ self.failure_reason += " missing offset {0}".format(i*4096)
128+
129+
130 class FioJobTest_iops_rate(FioJobTest):
131 """Test consists of fio test job t0009
132 Confirm that job0 iops == 1000
133@@ -889,6 +955,24 @@ TEST_LIST = [
134 'pre_success': None,
135 'requirements': [Requirements.linux, Requirements.io_uring],
136 },
137+ {
138+ 'test_id': 19,
139+ 'test_class': FioJobTest_t0019,
140+ 'job': 't0019.fio',
141+ 'success': SUCCESS_DEFAULT,
142+ 'pre_job': None,
143+ 'pre_success': None,
144+ 'requirements': [],
145+ },
146+ {
147+ 'test_id': 20,
148+ 'test_class': FioJobTest_t0020,
149+ 'job': 't0020.fio',
150+ 'success': SUCCESS_DEFAULT,
151+ 'pre_job': None,
152+ 'pre_success': None,
153+ 'requirements': [],
154+ },
155 {
156 'test_id': 1000,
157 'test_class': FioExeTest,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0028-test-use-Ubuntu-22.04-for-64-bit-tests.patch b/meta-oe/recipes-benchmark/fio/fio/0028-test-use-Ubuntu-22.04-for-64-bit-tests.patch
deleted file mode 100644
index b471d9c32..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0028-test-use-Ubuntu-22.04-for-64-bit-tests.patch
+++ /dev/null
@@ -1,72 +0,0 @@
1From abfe30b1fe8118a4ff935bd7cb03243329eba4b8 Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 29 Aug 2022 14:24:16 -0400
4Subject: [PATCH] test: use Ubuntu 22.04 for 64-bit tests
5
6On 22.04 there was a conflict among libunwind-14-dev, libunwind-dev, and
7libunwind8 that was resolved by removing libunwind-14-dev.
8
9The 32-bit Ubuntu setup steps require more attention to get them to work
10on 22.04. Stay on 20.04 for now and figure it out later.
11
12Starting pkgProblemResolver with broken count: 1
13Starting 2 pkgProblemResolver with broken count: 1
14Investigating (0) libunwind-14-dev:amd64 < 1:14.0.0-1ubuntu1 @ii K Ib >
15Broken libunwind-14-dev:amd64 Breaks on libunwind-dev:amd64 < none -> 1.3.2-2build2 @un puN >
16 Considering libunwind-dev:amd64 -1 as a solution to libunwind-14-dev:amd64 2
17Done
18Some packages could not be installed. This may mean that you have
19requested an impossible situation or if you are using the unstable
20distribution that some required packages have not yet been created
21or been moved out of Incoming.
22The following information may help to resolve the situation:
23
24The following packages have unmet dependencies:
25 libunwind-14-dev : Breaks: libunwind-dev but 1.3.2-2build2 is to be installed
26E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.
27
28Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
29---
30 .github/workflows/ci.yml | 6 +++---
31 ci/actions-install.sh | 2 ++
32 2 files changed, 5 insertions(+), 3 deletions(-)
33
34diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
35index 650366b2..85104e5a 100644
36--- a/.github/workflows/ci.yml
37+++ b/.github/workflows/ci.yml
38@@ -18,10 +18,10 @@ jobs:
39 - android
40 include:
41 - build: linux-gcc
42- os: ubuntu-20.04
43+ os: ubuntu-22.04
44 cc: gcc
45 - build: linux-clang
46- os: ubuntu-20.04
47+ os: ubuntu-22.04
48 cc: clang
49 - build: macos
50 os: macos-11
51@@ -29,7 +29,7 @@ jobs:
52 os: ubuntu-20.04
53 arch: i686
54 - build: android
55- os: ubuntu-20.04
56+ os: ubuntu-22.04
57 arch: aarch64-linux-android32
58
59 env:
60diff --git a/ci/actions-install.sh b/ci/actions-install.sh
61index b5c4198f..7017de2a 100755
62--- a/ci/actions-install.sh
63+++ b/ci/actions-install.sh
64@@ -54,6 +54,8 @@ DPKGCFG
65 libtcmalloc-minimal4
66 nvidia-cuda-dev
67 )
68+ echo "Removing libunwind-14-dev because of conflicts with libunwind-dev"
69+ sudo apt remove -y libunwind-14-dev
70 ;;
71 esac
72
diff --git a/meta-oe/recipes-benchmark/fio/fio/0029-test-get-32-bit-Ubuntu-22.04-build-working.patch b/meta-oe/recipes-benchmark/fio/fio/0029-test-get-32-bit-Ubuntu-22.04-build-working.patch
deleted file mode 100644
index 163ebf3b3..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0029-test-get-32-bit-Ubuntu-22.04-build-working.patch
+++ /dev/null
@@ -1,79 +0,0 @@
1From aa1075ba2ff300e4017bd7813423f63b1fbc325f Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Mon, 29 Aug 2022 15:15:56 -0400
4Subject: [PATCH] test: get 32-bit Ubuntu 22.04 build working
5
6Ubuntu 22.04 no longer has i386 builds for the packages libibverbs and
7librdmacm. So stop trying to install those packages for the 32-bit
8build.
9
10Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
11---
12 .github/workflows/ci.yml | 2 +-
13 ci/actions-install.sh | 11 ++++-------
14 2 files changed, 5 insertions(+), 8 deletions(-)
15
16diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
17index 85104e5a..bdc4db85 100644
18--- a/.github/workflows/ci.yml
19+++ b/.github/workflows/ci.yml
20@@ -26,7 +26,7 @@ jobs:
21 - build: macos
22 os: macos-11
23 - build: linux-i686-gcc
24- os: ubuntu-20.04
25+ os: ubuntu-22.04
26 arch: i686
27 - build: android
28 os: ubuntu-22.04
29diff --git a/ci/actions-install.sh b/ci/actions-install.sh
30index 7017de2a..c209a089 100755
31--- a/ci/actions-install.sh
32+++ b/ci/actions-install.sh
33@@ -23,26 +23,21 @@ DPKGCFG
34 libcunit1-dev
35 libcurl4-openssl-dev
36 libfl-dev
37- libibverbs-dev
38 libnuma-dev
39- librdmacm-dev
40 libnfs-dev
41 valgrind
42 )
43 case "${CI_TARGET_ARCH}" in
44 "i686")
45 sudo dpkg --add-architecture i386
46- opts="--allow-downgrades"
47 pkgs=("${pkgs[@]/%/:i386}")
48 pkgs+=(
49 gcc-multilib
50 pkg-config:i386
51 zlib1g-dev:i386
52- libpcre2-8-0=10.34-7
53 )
54 ;;
55 "x86_64")
56- opts=""
57 pkgs+=(
58 libglusterfs-dev
59 libgoogle-perftools-dev
60@@ -53,6 +48,8 @@ DPKGCFG
61 librbd-dev
62 libtcmalloc-minimal4
63 nvidia-cuda-dev
64+ libibverbs-dev
65+ librdmacm-dev
66 )
67 echo "Removing libunwind-14-dev because of conflicts with libunwind-dev"
68 sudo apt remove -y libunwind-14-dev
69@@ -68,8 +65,8 @@ DPKGCFG
70
71 echo "Updating APT..."
72 sudo apt-get -qq update
73- echo "Installing packages..."
74- sudo apt-get install "$opts" -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}"
75+ echo "Installing packages... ${pkgs[@]}"
76+ sudo apt-get install -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}"
77 }
78
79 install_linux() {
diff --git a/meta-oe/recipes-benchmark/fio/fio/0030-test-add-tests-for-lfsr-and-norandommap.patch b/meta-oe/recipes-benchmark/fio/fio/0030-test-add-tests-for-lfsr-and-norandommap.patch
deleted file mode 100644
index fa1d49e14..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0030-test-add-tests-for-lfsr-and-norandommap.patch
+++ /dev/null
@@ -1,143 +0,0 @@
1From 8d2b6305a6d0497bc6d78832be256380b1691694 Mon Sep 17 00:00:00 2001
2From: Vincent Fu <vincent.fu@samsung.com>
3Date: Tue, 30 Aug 2022 09:59:55 -0400
4Subject: [PATCH] test: add tests for lfsr and norandommap
5
6t0021 checks whether the lfsr random generator actually touches every
7offset.
8
9t0022 checks whether fio touches offsets more than once when
10norandommap=1.
11
12We should have automated tests for basic functionality to detect
13problems early.
14
15Signed-off-by: Vincent Fu <vincent.fu@samsung.com>
16---
17 t/jobs/t0021.fio | 15 +++++++++++++
18 t/jobs/t0022.fio | 13 +++++++++++
19 t/run-fio-tests.py | 55 +++++++++++++++++++++++++++++++++++++++++++++-
20 3 files changed, 82 insertions(+), 1 deletion(-)
21 create mode 100644 t/jobs/t0021.fio
22 create mode 100644 t/jobs/t0022.fio
23
24diff --git a/t/jobs/t0021.fio b/t/jobs/t0021.fio
25new file mode 100644
26index 00000000..47fbae71
27--- /dev/null
28+++ b/t/jobs/t0021.fio
29@@ -0,0 +1,15 @@
30+# make sure the lfsr random generator actually does touch all the offsets
31+#
32+# Expected result: offsets are not accessed sequentially and all offsets are touched
33+# Buggy result: offsets are accessed sequentially and one or more offsets are missed
34+# run with --debug=io or logging to see which offsets are read
35+
36+[test]
37+ioengine=null
38+filesize=1M
39+rw=randread
40+write_bw_log=test
41+per_job_logs=0
42+log_offset=1
43+norandommap=1
44+random_generator=lfsr
45diff --git a/t/jobs/t0022.fio b/t/jobs/t0022.fio
46new file mode 100644
47index 00000000..2324571e
48--- /dev/null
49+++ b/t/jobs/t0022.fio
50@@ -0,0 +1,13 @@
51+# make sure that when we enable norandommap we touch some offsets more than once
52+#
53+# Expected result: at least one offset is touched more than once
54+# Buggy result: each offset is touched only once
55+
56+[test]
57+ioengine=null
58+filesize=1M
59+rw=randread
60+write_bw_log=test
61+per_job_logs=0
62+log_offset=1
63+norandommap=1
64diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
65index 78f43521..47823761 100755
66--- a/t/run-fio-tests.py
67+++ b/t/run-fio-tests.py
68@@ -576,7 +576,7 @@ class FioJobTest_t0019(FioJobTest):
69
70
71 class FioJobTest_t0020(FioJobTest):
72- """Test consists of fio test job t0020
73+ """Test consists of fio test jobs t0020 and t0021
74 Confirm that almost all offsets were touched non-sequentially"""
75
76 def check_result(self):
77@@ -614,6 +614,41 @@ class FioJobTest_t0020(FioJobTest):
78 self.failure_reason += " missing offset {0}".format(i*4096)
79
80
81+class FioJobTest_t0022(FioJobTest):
82+ """Test consists of fio test job t0022"""
83+
84+ def check_result(self):
85+ super(FioJobTest_t0022, self).check_result()
86+
87+ bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
88+ file_data, success = self.get_file(bw_log_filename)
89+ log_lines = file_data.split('\n')
90+
91+ filesize = 1024*1024
92+ bs = 4096
93+ seq_count = 0
94+ offsets = set()
95+
96+ prev = int(log_lines[0].split(',')[4])
97+ for line in log_lines[1:]:
98+ offsets.add(prev/bs)
99+ if len(line.strip()) == 0:
100+ continue
101+ cur = int(line.split(',')[4])
102+ if cur - prev == bs:
103+ seq_count += 1
104+ prev = cur
105+
106+ # 10 is an arbitrary threshold
107+ if seq_count > 10:
108+ self.passed = False
109+ self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
110+
111+ if len(offsets) == filesize/bs:
112+ self.passed = False
113+ self.failure_reason += " no duplicate offsets found with norandommap=1".format(len(offsets))
114+
115+
116 class FioJobTest_iops_rate(FioJobTest):
117 """Test consists of fio test job t0009
118 Confirm that job0 iops == 1000
119@@ -973,6 +1008,24 @@ TEST_LIST = [
120 'pre_success': None,
121 'requirements': [],
122 },
123+ {
124+ 'test_id': 21,
125+ 'test_class': FioJobTest_t0020,
126+ 'job': 't0021.fio',
127+ 'success': SUCCESS_DEFAULT,
128+ 'pre_job': None,
129+ 'pre_success': None,
130+ 'requirements': [],
131+ },
132+ {
133+ 'test_id': 22,
134+ 'test_class': FioJobTest_t0022,
135+ 'job': 't0022.fio',
136+ 'success': SUCCESS_DEFAULT,
137+ 'pre_job': None,
138+ 'pre_success': None,
139+ 'requirements': [],
140+ },
141 {
142 'test_id': 1000,
143 'test_class': FioExeTest,
diff --git a/meta-oe/recipes-benchmark/fio/fio/0031-backend-revert-bad-memory-leak-fix.patch b/meta-oe/recipes-benchmark/fio/fio/0031-backend-revert-bad-memory-leak-fix.patch
deleted file mode 100644
index 335798cea..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0031-backend-revert-bad-memory-leak-fix.patch
+++ /dev/null
@@ -1,39 +0,0 @@
1From c060732180c981712f9a6fb7108c28a3c301c2c3 Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Tue, 30 Aug 2022 10:48:18 -0600
4Subject: [PATCH] backend: revert bad memory leak fix
5
6This essentially reverts the commit mentioned in the fixes line, as it
7causes crashes with using a trigger timeout + command.
8
9Fixes: 807473c36e10 ("fixed memory leak detected by ASAN")
10Signed-off-by: Jens Axboe <axboe@kernel.dk>
11---
12 backend.c | 5 -----
13 1 file changed, 5 deletions(-)
14
15diff --git a/backend.c b/backend.c
16index 375a23e4..fe614f6e 100644
17--- a/backend.c
18+++ b/backend.c
19@@ -2451,10 +2451,8 @@ reap:
20 strerror(ret));
21 } else {
22 pid_t pid;
23- struct fio_file **files;
24 void *eo;
25 dprint(FD_PROCESS, "will fork\n");
26- files = td->files;
27 eo = td->eo;
28 read_barrier();
29 pid = fork();
30@@ -2465,9 +2463,6 @@ reap:
31 _exit(ret);
32 } else if (i == fio_debug_jobno)
33 *fio_debug_jobp = pid;
34- // freeing previously allocated memory for files
35- // this memory freed MUST NOT be shared between processes, only the pointer itself may be shared within TD
36- free(files);
37 free(eo);
38 free(fd);
39 fd = NULL;
diff --git a/meta-oe/recipes-benchmark/fio/fio/0032-Fio-3.32.patch b/meta-oe/recipes-benchmark/fio/fio/0032-Fio-3.32.patch
deleted file mode 100644
index 6aa86d1fc..000000000
--- a/meta-oe/recipes-benchmark/fio/fio/0032-Fio-3.32.patch
+++ /dev/null
@@ -1,23 +0,0 @@
1From aaad03c6c8a9ef8cc0507a356a9fa2372e1f611b Mon Sep 17 00:00:00 2001
2From: Jens Axboe <axboe@kernel.dk>
3Date: Tue, 30 Aug 2022 10:51:13 -0600
4Subject: [PATCH] Fio 3.32
5
6Signed-off-by: Jens Axboe <axboe@kernel.dk>
7---
8 FIO-VERSION-GEN | 2 +-
9 1 file changed, 1 insertion(+), 1 deletion(-)
10
11diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN
12index 72630dd0..db073818 100755
13--- a/FIO-VERSION-GEN
14+++ b/FIO-VERSION-GEN
15@@ -1,7 +1,7 @@
16 #!/bin/sh
17
18 GVF=FIO-VERSION-FILE
19-DEF_VER=fio-3.31
20+DEF_VER=fio-3.32
21
22 LF='
23 '
diff --git a/meta-oe/recipes-benchmark/fio/fio_2022.bb b/meta-oe/recipes-benchmark/fio/fio_2022.bb
deleted file mode 100644
index 108b2b5c1..000000000
--- a/meta-oe/recipes-benchmark/fio/fio_2022.bb
+++ /dev/null
@@ -1,77 +0,0 @@
1SUMMARY = "Filesystem and hardware benchmark and stress tool"
2DESCRIPTION = "fio is an I/O tool meant to be used both for benchmark and \
3stress/hardware verification. It has support for a number of I/O engines, \
4I/O priorities (for newer Linux kernels), rate I/O, forked or threaded jobs, \
5and much more. It can work on block devices as well as files. fio accepts \
6job descriptions in a simple-to-understand text format. Several example job \
7files are included. fio displays all sorts of I/O performance information."
8HOMEPAGE = "http://freecode.com/projects/fio"
9SECTION = "console/tests"
10LICENSE = "GPL-2.0-only"
11LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263"
12
13DEPENDS = "libaio zlib coreutils-native"
14DEPENDS += "${@bb.utils.contains('MACHINE_FEATURES', 'pmem', 'pmdk', '', d)}"
15RDEPENDS:${PN} = "python3-core bash"
16
17PACKAGECONFIG_NUMA = "numa"
18# ARM does not currently support NUMA
19PACKAGECONFIG_NUMA:arm = ""
20PACKAGECONFIG_NUMA:armeb = ""
21
22PACKAGECONFIG ??= "${PACKAGECONFIG_NUMA}"
23PACKAGECONFIG[numa] = ",--disable-numa,numactl"
24
25SRCREV = "6e44f31b9241cdc56d0857fb10ddb2ec40faa541"
26SRC_URI = "git://git.kernel.dk/fio.git;branch=master \
27 file://0001-Fio-3.31.patch \
28 file://0002-lib-rand-Enhance-__fill_random_buf-using-the-multi-r.patch \
29 file://0003-lib-rand-get-rid-of-unused-MAX_SEED_BUCKETS.patch \
30 file://0004-ioengines-merge-filecreate-filestat-filedelete-engin.patch \
31 file://0005-engines-http-Add-storage-class-option-for-s3.patch \
32 file://0006-engines-http-Add-s3-crypto-options-for-s3.patch \
33 file://0007-doc-Add-usage-and-example-about-s3-storage-class-and.patch \
34 file://0008-README-link-to-GitHub-releases-for-Windows.patch \
35 file://0009-engines-xnvme-fix-segfault-issue-with-xnvme-ioengine.patch \
36 file://0010-doc-update-fio-doc-for-xnvme-engine.patch \
37 file://0011-test-add-latency-test-using-posixaio-ioengine.patch \
38 file://0012-test-fix-hash-for-t0016.patch \
39 file://0013-doc-get-rid-of-trailing-whitespace.patch \
40 file://0014-doc-clarify-that-I-O-errors-may-go-unnoticed-without.patch \
41 file://0015-Revert-Minor-style-fixups.patch \
42 file://0016-Revert-Fix-multithread-issues-when-operating-on-a-si.patch \
43 file://0017-Add-wait-for-handling-SIGBREAK.patch \
44 file://0018-engines-io_uring-pass-back-correct-error-value-when-.patch \
45 file://0019-Enable-CPU-affinity-support-on-Android.patch \
46 file://0020-io_uring-Replace-pthread_self-with-s-tid.patch \
47 file://0021-engines-io_uring-delete-debug-code.patch \
48 file://0022-t-io_uring-prep-for-including-engines-nvme.h-in-t-io.patch \
49 file://0023-t-io_uring-add-support-for-async-passthru.patch \
50 file://0024-t-io_uring-fix-64-bit-cast-on-32-bit-archs.patch \
51 file://0025-test-add-basic-test-for-io_uring-ioengine.patch \
52 file://0026-t-io_uring-remove-duplicate-definition-of-gettid.patch \
53 file://0027-test-add-some-tests-for-seq-and-rand-offsets.patch \
54 file://0028-test-use-Ubuntu-22.04-for-64-bit-tests.patch \
55 file://0029-test-get-32-bit-Ubuntu-22.04-build-working.patch \
56 file://0030-test-add-tests-for-lfsr-and-norandommap.patch \
57 file://0031-backend-revert-bad-memory-leak-fix.patch \
58 file://0032-Fio-3.32.patch \
59 "
60
61S = "${WORKDIR}/git"
62
63# avoids build breaks when using no-static-libs.inc
64DISABLE_STATIC = ""
65
66EXTRA_OEMAKE = "CC='${CC}' LDFLAGS='${LDFLAGS}'"
67EXTRA_OECONF = "${@bb.utils.contains('MACHINE_FEATURES', 'x86', '--disable-optimizations', '', d)}"
68
69do_configure() {
70 ./configure ${EXTRA_OECONF}
71}
72
73do_install() {
74 oe_runmake install DESTDIR=${D} prefix=${prefix} mandir=${mandir}
75 install -d ${D}/${docdir}/${PN}
76 cp -R --no-dereference --preserve=mode,links -v ${S}/examples ${D}/${docdir}/${PN}/
77}
diff --git a/meta-oe/recipes-benchmark/fio/fio_3.32.bb b/meta-oe/recipes-benchmark/fio/fio_3.32.bb
new file mode 100644
index 000000000..90e28340c
--- /dev/null
+++ b/meta-oe/recipes-benchmark/fio/fio_3.32.bb
@@ -0,0 +1,44 @@
1SUMMARY = "Filesystem and hardware benchmark and stress tool"
2DESCRIPTION = "fio is an I/O tool meant to be used both for benchmark and \
3stress/hardware verification. It has support for a number of I/O engines, \
4I/O priorities (for newer Linux kernels), rate I/O, forked or threaded jobs, \
5and much more. It can work on block devices as well as files. fio accepts \
6job descriptions in a simple-to-understand text format. Several example job \
7files are included. fio displays all sorts of I/O performance information."
8HOMEPAGE = "http://freecode.com/projects/fio"
9SECTION = "console/tests"
10LICENSE = "GPL-2.0-only"
11LIC_FILES_CHKSUM = "file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263"
12
13DEPENDS = "libaio zlib coreutils-native"
14DEPENDS += "${@bb.utils.contains('MACHINE_FEATURES', 'pmem', 'pmdk', '', d)}"
15RDEPENDS:${PN} = "python3-core bash"
16
17PACKAGECONFIG_NUMA = "numa"
18# ARM does not currently support NUMA
19PACKAGECONFIG_NUMA:arm = ""
20PACKAGECONFIG_NUMA:armeb = ""
21
22PACKAGECONFIG ??= "${PACKAGECONFIG_NUMA}"
23PACKAGECONFIG[numa] = ",--disable-numa,numactl"
24
25SRCREV = "db7fc8d864dc4fb607a0379333a0db60431bd649"
26SRC_URI = "git://git.kernel.dk/fio.git;branch=master"
27
28S = "${WORKDIR}/git"
29
30# avoids build breaks when using no-static-libs.inc
31DISABLE_STATIC = ""
32
33EXTRA_OEMAKE = "CC='${CC}' LDFLAGS='${LDFLAGS}'"
34EXTRA_OECONF = "${@bb.utils.contains('MACHINE_FEATURES', 'x86', '--disable-optimizations', '', d)}"
35
36do_configure() {
37 ./configure ${EXTRA_OECONF}
38}
39
40do_install() {
41 oe_runmake install DESTDIR=${D} prefix=${prefix} mandir=${mandir}
42 install -d ${D}/${docdir}/${PN}
43 cp -R --no-dereference --preserve=mode,links -v ${S}/examples ${D}/${docdir}/${PN}/
44}
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-11-17 16:20:59 +0000