diff options
4 files changed, 265 insertions, 6 deletions
diff --git a/meta/recipes-extended/lighttpd/lighttpd/0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch b/meta/recipes-extended/lighttpd/lighttpd/0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch index f17bdce2c0..44b9136b05 100644 --- a/meta/recipes-extended/lighttpd/lighttpd/0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch +++ b/meta/recipes-extended/lighttpd/lighttpd/0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch | |||
@@ -1,4 +1,4 @@ | |||
1 | From 22afc5d9aaa215c3c87ba21c77d47da44ab3b113 Mon Sep 17 00:00:00 2001 | 1 | From f918d5ba6ff1d439822be063237aea2705ea27b8 Mon Sep 17 00:00:00 2001 |
2 | From: Alexander Kanavin <alex.kanavin@gmail.com> | 2 | From: Alexander Kanavin <alex.kanavin@gmail.com> |
3 | Date: Fri, 26 Aug 2016 18:20:32 +0300 | 3 | Date: Fri, 26 Aug 2016 18:20:32 +0300 |
4 | Subject: [PATCH] Use pkg-config for pcre dependency instead of -config script. | 4 | Subject: [PATCH] Use pkg-config for pcre dependency instead of -config script. |
@@ -6,15 +6,16 @@ Subject: [PATCH] Use pkg-config for pcre dependency instead of -config script. | |||
6 | RP 2014/5/22 | 6 | RP 2014/5/22 |
7 | Upstream-Status: Pending | 7 | Upstream-Status: Pending |
8 | Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> | 8 | Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> |
9 | |||
9 | --- | 10 | --- |
10 | configure.ac | 16 ++++++++++++---- | 11 | configure.ac | 16 ++++++++++++---- |
11 | 1 file changed, 12 insertions(+), 4 deletions(-) | 12 | 1 file changed, 12 insertions(+), 4 deletions(-) |
12 | 13 | ||
13 | diff --git a/configure.ac b/configure.ac | 14 | diff --git a/configure.ac b/configure.ac |
14 | index 5383cec..c29a902 100644 | 15 | index dbddfb9..62cf17f 100644 |
15 | --- a/configure.ac | 16 | --- a/configure.ac |
16 | +++ b/configure.ac | 17 | +++ b/configure.ac |
17 | @@ -651,10 +651,18 @@ AC_ARG_WITH([pcre], | 18 | @@ -748,10 +748,18 @@ AC_ARG_WITH([pcre], |
18 | ) | 19 | ) |
19 | AC_MSG_RESULT([$WITH_PCRE]) | 20 | AC_MSG_RESULT([$WITH_PCRE]) |
20 | 21 | ||
@@ -37,6 +38,3 @@ index 5383cec..c29a902 100644 | |||
37 | else | 38 | else |
38 | AC_PATH_PROG([PCRECONFIG], [pcre-config]) | 39 | AC_PATH_PROG([PCRECONFIG], [pcre-config]) |
39 | if test -n "$PCRECONFIG"; then | 40 | if test -n "$PCRECONFIG"; then |
40 | -- | ||
41 | 2.15.0 | ||
42 | |||
diff --git a/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch new file mode 100644 index 0000000000..e226366112 --- /dev/null +++ b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch | |||
@@ -0,0 +1,224 @@ | |||
1 | From a566fe4cc9f9d0ef9cfdcbc13159ef0644e91c9c Mon Sep 17 00:00:00 2001 | ||
2 | From: Glenn Strauss <gstrauss@gluelogic.com> | ||
3 | Date: Wed, 23 Dec 2020 23:14:47 -0500 | ||
4 | Subject: [PATCH] reuse large mem chunks (fix mem usage) (fixes #3033) | ||
5 | |||
6 | (cherry picked from commit 7ba521ffb4959f6f74a609d5d4acafc29a038337) | ||
7 | |||
8 | (thx flynn) | ||
9 | |||
10 | fix large memory usage for large file downloads from dynamic backends | ||
11 | |||
12 | reuse or release large memory chunks | ||
13 | |||
14 | x-ref: | ||
15 | "Memory Growth with PUT and full buffered streams" | ||
16 | https://redmine.lighttpd.net/issues/3033 | ||
17 | |||
18 | Upstream-Status: Backport | ||
19 | Comment: Hunk refreshed to make it backword compatible. | ||
20 | https://redmine.lighttpd.net/projects/lighttpd/repository/14/revisions/7ba521ffb4959f6f74a609d5d4acafc29a038337 | ||
21 | Signed-off-by: Purushottam Choudhary <Purushottam.Choudhary@kpit.com> | ||
22 | |||
23 | --- | ||
24 | src/chunk.c | 99 +++++++++++++++++++++++++++++++++--------- | ||
25 | src/chunk.h | 2 + | ||
26 | src/http-header-glue.c | 2 +- | ||
27 | 3 files changed, 82 insertions(+), 21 deletions(-) | ||
28 | |||
29 | diff --git a/src/chunk.c b/src/chunk.c | ||
30 | index 133308f..d7259b9 100644 | ||
31 | --- a/src/chunk.c | ||
32 | +++ b/src/chunk.c | ||
33 | @@ -28,16 +28,20 @@ | ||
34 | static size_t chunk_buf_sz = 8192; | ||
35 | static chunk *chunks, *chunks_oversized; | ||
36 | static chunk *chunk_buffers; | ||
37 | +static int chunks_oversized_n; | ||
38 | static array *chunkqueue_default_tempdirs = NULL; | ||
39 | static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; | ||
40 | |||
41 | void chunkqueue_set_chunk_size (size_t sz) | ||
42 | { | ||
43 | - chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 8192; | ||
44 | + size_t x = 1024; | ||
45 | + while (x < sz && x < (1u << 30)) x <<= 1; | ||
46 | + chunk_buf_sz = sz > 0 ? x : 8192; | ||
47 | } | ||
48 | |||
49 | void chunkqueue_set_tempdirs_default_reset (void) | ||
50 | { | ||
51 | + chunk_buf_sz = 8192; | ||
52 | chunkqueue_default_tempdirs = NULL; | ||
53 | chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; | ||
54 | } | ||
55 | @@ -120,15 +124,49 @@ static void chunk_free(chunk *c) { | ||
56 | free(c); | ||
57 | } | ||
58 | |||
59 | -buffer * chunk_buffer_acquire(void) { | ||
60 | +static chunk * chunk_pop_oversized(size_t sz) { | ||
61 | + /* future: might have buckets of certain sizes, up to socket buf sizes */ | ||
62 | + if (chunks_oversized && chunks_oversized->mem->size >= sz) { | ||
63 | + --chunks_oversized_n; | ||
64 | + chunk *c = chunks_oversized; | ||
65 | + chunks_oversized = c->next; | ||
66 | + return c; | ||
67 | + } | ||
68 | + return NULL; | ||
69 | +} | ||
70 | + | ||
71 | +static void chunk_push_oversized(chunk * const c, const size_t sz) { | ||
72 | + if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) { | ||
73 | + ++chunks_oversized_n; | ||
74 | + chunk **co = &chunks_oversized; | ||
75 | + while (*co && sz < (*co)->mem->size) co = &(*co)->next; | ||
76 | + c->next = *co; | ||
77 | + *co = c; | ||
78 | + } | ||
79 | + else | ||
80 | + chunk_free(c); | ||
81 | +} | ||
82 | + | ||
83 | +static buffer * chunk_buffer_acquire_sz(size_t sz) { | ||
84 | chunk *c; | ||
85 | buffer *b; | ||
86 | - if (chunks) { | ||
87 | - c = chunks; | ||
88 | - chunks = c->next; | ||
89 | + if (sz <= chunk_buf_sz) { | ||
90 | + if (chunks) { | ||
91 | + c = chunks; | ||
92 | + chunks = c->next; | ||
93 | + } | ||
94 | + else | ||
95 | + c = chunk_init(chunk_buf_sz); | ||
96 | + /* future: might choose to pop from chunks_oversized, if available | ||
97 | + * (even if larger than sz) rather than allocating new chunk | ||
98 | + * (and if doing so, might replace chunks_oversized_n) */ | ||
99 | } | ||
100 | else { | ||
101 | - c = chunk_init(chunk_buf_sz); | ||
102 | + /*(round up to nearest chunk_buf_sz)*/ | ||
103 | + sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); | ||
104 | + c = chunk_pop_oversized(sz); | ||
105 | + if (NULL == c) | ||
106 | + c = chunk_init(sz); | ||
107 | } | ||
108 | c->next = chunk_buffers; | ||
109 | chunk_buffers = c; | ||
110 | @@ -137,21 +175,47 @@ buffer * chunk_buffer_acquire(void) { | ||
111 | return b; | ||
112 | } | ||
113 | |||
114 | +buffer * chunk_buffer_acquire(void) { | ||
115 | + return chunk_buffer_acquire_sz(chunk_buf_sz); | ||
116 | +} | ||
117 | + | ||
118 | void chunk_buffer_release(buffer *b) { | ||
119 | if (NULL == b) return; | ||
120 | - if (b->size >= chunk_buf_sz && chunk_buffers) { | ||
121 | + if (chunk_buffers) { | ||
122 | chunk *c = chunk_buffers; | ||
123 | chunk_buffers = c->next; | ||
124 | c->mem = b; | ||
125 | - c->next = chunks; | ||
126 | - chunks = c; | ||
127 | buffer_clear(b); | ||
128 | + if (b->size == chunk_buf_sz) { | ||
129 | + c->next = chunks; | ||
130 | + chunks = c; | ||
131 | + } | ||
132 | + else if (b->size > chunk_buf_sz) | ||
133 | + chunk_push_oversized(c, b->size); | ||
134 | + else | ||
135 | + chunk_free(c); | ||
136 | } | ||
137 | else { | ||
138 | buffer_free(b); | ||
139 | } | ||
140 | } | ||
141 | |||
142 | +size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) { | ||
143 | + if (sz > chunk_buffer_string_space(b)) { | ||
144 | + sz += b->used ? b->used : 1; | ||
145 | + buffer * const cb = chunk_buffer_acquire_sz(sz); | ||
146 | + /* swap buffer contents and copy original b->ptr into larger b->ptr */ | ||
147 | + /*(this does more than buffer_move())*/ | ||
148 | + buffer tb = *b; | ||
149 | + *b = *cb; | ||
150 | + *cb = tb; | ||
151 | + if ((b->used = tb.used)) | ||
152 | + memcpy(b->ptr, tb.ptr, tb.used); | ||
153 | + chunk_buffer_release(cb); | ||
154 | + } | ||
155 | + return chunk_buffer_string_space(b); | ||
156 | +} | ||
157 | + | ||
158 | static chunk * chunk_acquire(size_t sz) { | ||
159 | if (sz <= chunk_buf_sz) { | ||
160 | if (chunks) { | ||
161 | @@ -162,13 +226,10 @@ static chunk * chunk_acquire(size_t sz) { | ||
162 | sz = chunk_buf_sz; | ||
163 | } | ||
164 | else { | ||
165 | - sz = (sz + 8191) & ~8191uL; | ||
166 | - /* future: might have buckets of certain sizes, up to socket buf sizes*/ | ||
167 | - if (chunks_oversized && chunks_oversized->mem->size >= sz) { | ||
168 | - chunk *c = chunks_oversized; | ||
169 | - chunks_oversized = c->next; | ||
170 | - return c; | ||
171 | - } | ||
172 | + /*(round up to nearest chunk_buf_sz)*/ | ||
173 | + sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); | ||
174 | + chunk *c = chunk_pop_oversized(sz); | ||
175 | + if (c) return c; | ||
176 | } | ||
177 | |||
178 | return chunk_init(sz); | ||
179 | @@ -183,10 +244,7 @@ static void chunk_release(chunk *c) { | ||
180 | } | ||
181 | else if (sz > chunk_buf_sz) { | ||
182 | chunk_reset(c); | ||
183 | - chunk **co = &chunks_oversized; | ||
184 | - while (*co && sz < (*co)->mem->size) co = &(*co)->next; | ||
185 | - c->next = *co; | ||
186 | - *co = c; | ||
187 | + chunk_push_oversized(c, sz); | ||
188 | } | ||
189 | else { | ||
190 | chunk_free(c); | ||
191 | @@ -205,6 +263,7 @@ void chunkqueue_chunk_pool_clear(void) | ||
192 | chunk_free(c); | ||
193 | } | ||
194 | chunks_oversized = NULL; | ||
195 | + chunks_oversized_n = 0; | ||
196 | } | ||
197 | |||
198 | void chunkqueue_chunk_pool_free(void) | ||
199 | diff --git a/src/chunk.h b/src/chunk.h | ||
200 | index 4c6b7e4..93f343c 100644 | ||
201 | --- a/src/chunk.h | ||
202 | +++ b/src/chunk.h | ||
203 | @@ -50,6 +50,8 @@ typedef struct { | ||
204 | buffer * chunk_buffer_acquire(void); | ||
205 | void chunk_buffer_release(buffer *b); | ||
206 | |||
207 | +size_t chunk_buffer_prepare_append (buffer *b, size_t sz); | ||
208 | + | ||
209 | void chunkqueue_chunk_pool_clear(void); | ||
210 | void chunkqueue_chunk_pool_free(void); | ||
211 | |||
212 | diff --git a/src/http-header-glue.c b/src/http-header-glue.c | ||
213 | index d54f00c..2231fba 100644 | ||
214 | --- a/src/http-header-glue.c | ||
215 | +++ b/src/http-header-glue.c | ||
216 | @@ -1267,7 +1267,7 @@ handler_t http_response_read(server *srv, connection *con, http_response_opts *o | ||
217 | if (avail < toread) { | ||
218 | /*(add avail+toread to reduce allocations when ioctl EOPNOTSUPP)*/ | ||
219 | avail = avail ? avail - 1 + toread : toread; | ||
220 | - buffer_string_prepare_append(b, avail); | ||
221 | + avail = chunk_buffer_prepare_append(b, avail); | ||
222 | } | ||
223 | |||
224 | n = read(fd, b->ptr+buffer_string_length(b), avail); | ||
diff --git a/meta/recipes-extended/lighttpd/lighttpd/default-chunk-size-8k.patch b/meta/recipes-extended/lighttpd/lighttpd/default-chunk-size-8k.patch new file mode 100644 index 0000000000..fd75ca6e26 --- /dev/null +++ b/meta/recipes-extended/lighttpd/lighttpd/default-chunk-size-8k.patch | |||
@@ -0,0 +1,35 @@ | |||
1 | From 2e08ee1d404e308f15551277e92b7605ddfa96a8 Mon Sep 17 00:00:00 2001 | ||
2 | From: Glenn Strauss <gstrauss@gluelogic.com> | ||
3 | Date: Fri, 29 Nov 2019 18:18:52 -0500 | ||
4 | Subject: [PATCH] default chunk size 8k (was 4k) | ||
5 | |||
6 | Upstream-Status: Backport | ||
7 | Comment: No hunk refreshed | ||
8 | https://git.lighttpd.net/lighttpd/lighttpd1.4/commit/304e46d4f808c46cbb025edfacf2913a30ce8855 | ||
9 | Signed-off-by: Purushottam Choudhary <Purushottam.Choudhary@kpit.com> | ||
10 | --- | ||
11 | src/chunk.c | 4 ++-- | ||
12 | 1 file changed, 2 insertions(+), 2 deletions(-) | ||
13 | |||
14 | diff --git a/src/chunk.c b/src/chunk.c | ||
15 | index 09dd3f1..133308f 100644 | ||
16 | --- a/src/chunk.c | ||
17 | +++ b/src/chunk.c | ||
18 | @@ -25,7 +25,7 @@ | ||
19 | #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024) | ||
20 | #define MAX_TEMPFILE_SIZE (128 * 1024 * 1024) | ||
21 | |||
22 | -static size_t chunk_buf_sz = 4096; | ||
23 | +static size_t chunk_buf_sz = 8192; | ||
24 | static chunk *chunks, *chunks_oversized; | ||
25 | static chunk *chunk_buffers; | ||
26 | static array *chunkqueue_default_tempdirs = NULL; | ||
27 | @@ -33,7 +33,7 @@ static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; | ||
28 | |||
29 | void chunkqueue_set_chunk_size (size_t sz) | ||
30 | { | ||
31 | - chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 4096; | ||
32 | + chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 8192; | ||
33 | } | ||
34 | |||
35 | void chunkqueue_set_tempdirs_default_reset (void) | ||
diff --git a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb index 35a268a03f..737d6ebf7c 100644 --- a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb +++ b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb | |||
@@ -18,6 +18,8 @@ SRC_URI = "http://download.lighttpd.net/lighttpd/releases-1.4.x/lighttpd-${PV}.t | |||
18 | file://lighttpd.conf \ | 18 | file://lighttpd.conf \ |
19 | file://lighttpd \ | 19 | file://lighttpd \ |
20 | file://0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch \ | 20 | file://0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch \ |
21 | file://default-chunk-size-8k.patch \ | ||
22 | file://0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch \ | ||
21 | " | 23 | " |
22 | 24 | ||
23 | SRC_URI[md5sum] = "be4bda2c28bcbdac6eb941528f6edf03" | 25 | SRC_URI[md5sum] = "be4bda2c28bcbdac6eb941528f6edf03" |