Memory leak from timings

Memory leak happens when the dynamic list times grows without anything to
reset it.

In ceilometer and cinder configuration files, the new option is created:
[nova_client]
max_timing_buffer=<value>

In all clients found that uses extends the HTTPClient and uses the times
list, we limit the size of the list by popping off the oldest item in the
list to maintain a maximum size. A default size of 200 is chosen,
configurable by the above configuration option.

Signed-off-by: Amy Fong <amy.fong@windriver.com>
Signed-off-by: Bruce Ashfield <bruce.ashfield@windriver.com>

18 files changed, 374 insertions, 7 deletions

diff --git a/meta-openstack/recipes-devtools/python/python-ceilometer/ceilometer.conf b/meta-openstack/recipes-devtools/python/python-ceilometer/ceilometer.conf
index 4632f40..6340a15 100644
--- a/meta-openstack/recipes-devtools/python/python-ceilometer/ceilometer.conf
+++ b/meta-openstack/recipes-devtools/python/python-ceilometer/ceilometer.conf
@@ -868,4 +868,5 @@ admin_tenant_name=%SERVICE_TENANT_NAME%
 # Password for Redis server. (optional) (string value)
 #password=<None>
 
-
+# [nova_client]
+# max_timing_buffer=100
diff --git a/meta-openstack/recipes-devtools/python/python-ceilometer/fix_ceilometer_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-ceilometer/fix_ceilometer_memory_leak.patch
new file mode 100644
index 0000000..bdc1bf7
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-ceilometer/fix_ceilometer_memory_leak.patch
@@ -0,0 +1,27 @@
+---
+ ceilometer/nova_client.py |    7 +++++++
+ 1 file changed, 7 insertions(+)
+
+--- a/ceilometer/nova_client.py
++++ b/ceilometer/nova_client.py
+@@ -26,6 +26,12 @@
+ 
+ LOG = log.getLogger(__name__)
+ 
++nova_client_opts = [
++    cfg.IntOpt('max_timing_buffer',
++               default=200,
++               help='The max number of the timing objects to keep'),
++]
++cfg.CONF.register_opts(nova_client_opts, group="nova_client")
+ 
+ def logged(func):
+ 
+@@ -56,6 +62,7 @@
+             endpoint_type=conf.os_endpoint_type,
+             cacert=conf.os_cacert,
+             no_cache=True)
++        self.nova_client.set_timings_max_len(cfg.CONF.nova_client.max_timing_buffer)
+ 
+     def _with_flavor_and_image(self, instances):
+         for instance in instances:
diff --git a/meta-openstack/recipes-devtools/python/python-ceilometer_git.bb b/meta-openstack/recipes-devtools/python/python-ceilometer_git.bb
index 7d1796d..f77cf1d 100644
--- a/meta-openstack/recipes-devtools/python/python-ceilometer_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-ceilometer_git.bb
@@ -11,6 +11,7 @@ SRC_URI = "git://github.com/openstack/${SRCNAME}.git;branch=master \
            file://ceilometer.conf \
            file://ceilometer.init \
            file://0001-sqlalchemy-Fix-for-get_statistics-with-postgresql.patch \
+           file://fix_ceilometer_memory_leak.patch \
 "
 
 SRCREV="a4c7411ac903984c7e7524469f89a417cf9cf97e"
diff --git a/meta-openstack/recipes-devtools/python/python-ceilometerclient/fix_ceilometerclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-ceilometerclient/fix_ceilometerclient_memory_leak.patch
new file mode 100644
index 0000000..8abc654
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-ceilometerclient/fix_ceilometerclient_memory_leak.patch
@@ -0,0 +1,37 @@
+---
+ ceilometerclient/openstack/common/apiclient/client.py |   12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+
+--- a/ceilometerclient/openstack/common/apiclient/client.py
++++ b/ceilometerclient/openstack/common/apiclient/client.py
+@@ -90,6 +90,7 @@
+         self.user_agent = user_agent or self.user_agent
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+         self.timings = timings
+ 
+         # requests within the same session can reuse TCP connections from pool
+@@ -142,6 +143,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def request(self, method, url, **kwargs):
+         """Send an http request with the specified characteristics.
+ 
+@@ -173,6 +180,9 @@
+         if self.timings:
+             self.times.append(("%s %s" % (method, url),
+                                start_time, time.time()))
++            # remove oldest items until we maintain max length
++            while len(self.times) > self.times_max_len:
++                del self.times[0]
+         self._http_log_resp(resp)
+ 
+         if resp.status_code >= 400:
diff --git a/meta-openstack/recipes-devtools/python/python-ceilometerclient_git.bb b/meta-openstack/recipes-devtools/python/python-ceilometerclient_git.bb
index 811bbd5..c7c6ff4 100644
--- a/meta-openstack/recipes-devtools/python/python-ceilometerclient_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-ceilometerclient_git.bb
@@ -20,7 +20,10 @@ RDEPENDS_${PN} +="python-cliff \
 
 PR = "r0"
 
-SRC_URI = "git://github.com/openstack/python-ceilometerclient.git;branch=master"
+SRC_URI = "\
+        git://github.com/openstack/python-ceilometerclient.git;branch=master \
+        file://fix_ceilometerclient_memory_leak.patch \
+        "
 
 PV="1.0.9+git${SRCPV}"
 SRCREV="6f4ec9dc96477131c90040ee00f5535e50d5f914"
diff --git a/meta-openstack/recipes-devtools/python/python-cinder/cinder.conf b/meta-openstack/recipes-devtools/python/python-cinder/cinder.conf
index fff6910..99a0fc7 100644
--- a/meta-openstack/recipes-devtools/python/python-cinder/cinder.conf
+++ b/meta-openstack/recipes-devtools/python/python-cinder/cinder.conf
@@ -1218,3 +1218,6 @@ rbd_user=cinder-volume
 volume_backend_name=RBD_CEPH
 
 # Total option count: 255
+
+# [nova_client]
+# max_timing_buffer=100
diff --git a/meta-openstack/recipes-devtools/python/python-cinder/fix_cinder_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-cinder/fix_cinder_memory_leak.patch
new file mode 100644
index 0000000..2566531
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-cinder/fix_cinder_memory_leak.patch
@@ -0,0 +1,28 @@
+---
+ cinder/compute/nova.py |    8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+--- a/cinder/compute/nova.py
++++ b/cinder/compute/nova.py
+@@ -57,6 +57,13 @@
+ CONF = cfg.CONF
+ CONF.register_opts(nova_opts)
+ 
++nova_client_opts = [
++    cfg.IntOpt('max_timing_buffer',
++               default=200,
++               help='The max number of the timing objects to keep'),
++]
++cfg.CONF.register_opts(nova_client_opts, group="nova_client")
++
+ LOG = logging.getLogger(__name__)
+ 
+ 
+@@ -106,6 +113,7 @@
+                            insecure=CONF.nova_api_insecure,
+                            cacert=CONF.nova_ca_certificates_file,
+                            extensions=extensions)
++    c.set_timings_max_len(cfg.CONF.nova_client.max_timing_buffer)
+     # noauth extracts user_id:project_id from auth_token
+     c.client.auth_token = context.auth_token or '%s:%s' % (context.user_id,
+                                                            context.project_id)
diff --git a/meta-openstack/recipes-devtools/python/python-cinder_git.bb b/meta-openstack/recipes-devtools/python/python-cinder_git.bb
index 8b08dbb..15a2608 100644
--- a/meta-openstack/recipes-devtools/python/python-cinder_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-cinder_git.bb
@@ -16,6 +16,7 @@ SRC_URI = "git://github.com/openstack/${SRCNAME}.git;branch=stable/havana \
     file://glusterfs_setup.sh \
     file://lvm_iscsi_setup.sh \
     file://add-cinder-volume-types.sh \
+    file://fix_cinder_memory_leak.patch \
         "
 
 SRCREV="8b5fb8409322f61d8b610c97c109a61bf48a940e"
diff --git a/meta-openstack/recipes-devtools/python/python-cinderclient/fix_cinderclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-cinderclient/fix_cinderclient_memory_leak.patch
new file mode 100644
index 0000000..137edfd
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-cinderclient/fix_cinderclient_memory_leak.patch
@@ -0,0 +1,37 @@
+---
+ cinderclient/openstack/common/apiclient/client.py |   10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+--- a/cinderclient/openstack/common/apiclient/client.py
++++ b/cinderclient/openstack/common/apiclient/client.py
+@@ -90,6 +90,7 @@
+         self.user_agent = user_agent or self.user_agent
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+         self.timings = timings
+ 
+         # requests within the same session can reuse TCP connections from pool
+@@ -142,6 +143,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def request(self, method, url, **kwargs):
+         """Send an http request with the specified characteristics.
+ 
+@@ -173,6 +180,9 @@
+         if self.timings:
+             self.times.append(("%s %s" % (method, url),
+                                start_time, time.time()))
++            # remove oldest items until we maintain max length
++            while len(self.times) > self.times_max_len:
++                del self.times[0]
+         self._http_log_resp(resp)
+ 
+         if resp.status_code >= 400:
diff --git a/meta-openstack/recipes-devtools/python/python-cinderclient_git.bb b/meta-openstack/recipes-devtools/python/python-cinderclient_git.bb
index f12975e..3c7293b 100644
--- a/meta-openstack/recipes-devtools/python/python-cinderclient_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-cinderclient_git.bb
@@ -8,7 +8,10 @@ DEPENDS = "python-setuptools-git"
 PR = "r0"
 SRCNAME = "python-cinderclient"
 
-SRC_URI = "git://github.com/openstack/python-cinderclient.git;branch=master"
+SRC_URI = "\
+        git://github.com/openstack/python-cinderclient.git;branch=master \
+        file://fix_cinderclient_memory_leak.patch \
+        "
 
 PV="1.0.8+git${SRCPV}"
 SRCREV="ea8c9554c947b7fd30adfb1249a9317f258901ab"
diff --git a/meta-openstack/recipes-devtools/python/python-glanceclient/fix_glanceclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-glanceclient/fix_glanceclient_memory_leak.patch
new file mode 100644
index 0000000..754d83f
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-glanceclient/fix_glanceclient_memory_leak.patch
@@ -0,0 +1,37 @@
+---
+ glanceclient/openstack/common/apiclient/client.py |   10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+--- a/glanceclient/openstack/common/apiclient/client.py
++++ b/glanceclient/openstack/common/apiclient/client.py
+@@ -90,6 +90,7 @@
+         self.user_agent = user_agent or self.user_agent
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+         self.timings = timings
+ 
+         # requests within the same session can reuse TCP connections from pool
+@@ -142,6 +143,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def request(self, method, url, **kwargs):
+         """Send an http request with the specified characteristics.
+ 
+@@ -173,6 +180,9 @@
+         if self.timings:
+             self.times.append(("%s %s" % (method, url),
+                                start_time, time.time()))
++            # remove oldest items until we maintain max length
++            while len(self.times) > self.times_max_len:
++                del self.times[0]
+         self._http_log_resp(resp)
+ 
+         if resp.status_code >= 400:
diff --git a/meta-openstack/recipes-devtools/python/python-glanceclient_git.bb b/meta-openstack/recipes-devtools/python/python-glanceclient_git.bb
index 1d7a4d1..7777f99 100644
--- a/meta-openstack/recipes-devtools/python/python-glanceclient_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-glanceclient_git.bb
@@ -13,7 +13,10 @@ SRCREV = "23ad1d6db7ec8116afed7c8aca3832dac440930a"
 PV = "0.12.0+gitr${SRCREV}"
 PR = "0"
 
-SRC_URI = "git://github.com/openstack/${PN}.git;protocol=https"
+SRC_URI = "\
+        git://github.com/openstack/${PN}.git;protocol=https \
+        file://fix_glanceclient_memory_leak.patch \
+        "
 
 S = "${WORKDIR}/git"
 
diff --git a/meta-openstack/recipes-devtools/python/python-hp3parclient/fix_hp3parclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-hp3parclient/fix_hp3parclient_memory_leak.patch
new file mode 100644
index 0000000..ac196fd
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-hp3parclient/fix_hp3parclient_memory_leak.patch
@@ -0,0 +1,37 @@
+---
+ hp3parclient/http.py |   10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+--- a/hp3parclient/http.py
++++ b/hp3parclient/http.py
+@@ -61,6 +61,7 @@
+         self.set_debug_flag(http_log_debug)
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+ 
+         # httplib2 overrides
+         self.force_exception_to_status_code = True
+@@ -141,6 +142,12 @@
+         """
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def _http_log_req(self, args, kwargs):
+         if not self.http_log_debug:
+             return
+@@ -205,6 +212,9 @@
+         resp, body = self.request(url, method, **kwargs)
+         self.times.append(("%s %s" % (method, url),
+                            start_time, time.time()))
++        # remove oldest items until we maintain max length
++        while len(self.times) > self.times_max_len:
++            del self.times[0]
+         return resp, body
+ 
+ 
diff --git a/meta-openstack/recipes-devtools/python/python-hp3parclient_2.0.0.bb b/meta-openstack/recipes-devtools/python/python-hp3parclient_2.0.0.bb
index 75ad641..9be086f 100644
--- a/meta-openstack/recipes-devtools/python/python-hp3parclient_2.0.0.bb
+++ b/meta-openstack/recipes-devtools/python/python-hp3parclient_2.0.0.bb
@@ -7,7 +7,10 @@ LIC_FILES_CHKSUM = "file://LICENSE.txt;md5=3b83ef96387f14655fc854ddc3c6bd57"
 PR = "r0"
 SRCNAME = "hp3parclient"
 
-SRC_URI = "https://pypi.python.org/packages/source/h/${SRCNAME}/${SRCNAME}-${PV}.tar.gz"
+SRC_URI = "\
+        https://pypi.python.org/packages/source/h/${SRCNAME}/${SRCNAME}-${PV}.tar.gz \
+        file://fix_hp3parclient_memory_leak.patch \
+        "
 
 SRC_URI[md5sum] = "52cdcb930eb0dc81d93689abe169c502"
 SRC_URI[sha256sum] = "bab78a3fa14236d7884af52271fd91d08d51ec99b7e2d6c6fb477bfdfb9ef259"
diff --git a/meta-openstack/recipes-devtools/python/python-keystoneclient/fix_keystoneclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-keystoneclient/fix_keystoneclient_memory_leak.patch
new file mode 100644
index 0000000..05a1d23
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-keystoneclient/fix_keystoneclient_memory_leak.patch
@@ -0,0 +1,37 @@
+---
+ keystoneclient/openstack/common/apiclient/client.py |   10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+--- a/keystoneclient/openstack/common/apiclient/client.py
++++ b/keystoneclient/openstack/common/apiclient/client.py
+@@ -90,6 +90,7 @@
+         self.user_agent = user_agent or self.user_agent
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+         self.timings = timings
+ 
+         # requests within the same session can reuse TCP connections from pool
+@@ -142,6 +143,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def request(self, method, url, **kwargs):
+         """Send an http request with the specified characteristics.
+ 
+@@ -173,6 +180,9 @@
+         if self.timings:
+             self.times.append(("%s %s" % (method, url),
+                                start_time, time.time()))
++            # remove oldest items until we maintain max length
++            while len(self.times) > self.times_max_len:
++                del self.times[0]
+         self._http_log_resp(resp)
+ 
+         if resp.status_code >= 400:
diff --git a/meta-openstack/recipes-devtools/python/python-keystoneclient_git.bb b/meta-openstack/recipes-devtools/python/python-keystoneclient_git.bb
index be2395a..89d8a9a 100644
--- a/meta-openstack/recipes-devtools/python/python-keystoneclient_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-keystoneclient_git.bb
@@ -6,7 +6,10 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=4a4d0e932ffae1c0131528d30d419c55"
 
 PR = "r0"
 
-SRC_URI = "git://github.com/openstack/python-keystoneclient.git;branch=master"
+SRC_URI = "\
+        git://github.com/openstack/python-keystoneclient.git;branch=master \
+        file://fix_keystoneclient_memory_leak.patch \
+        "
 
 PV="0.6.0+git${SRCPV}"
 SRCREV="a6b8e506740935498f5eba319c67b740d17752a2"
diff --git a/meta-openstack/recipes-devtools/python/python-novaclient/fix_novaclient_memory_leak.patch b/meta-openstack/recipes-devtools/python/python-novaclient/fix_novaclient_memory_leak.patch
new file mode 100644
index 0000000..5732d9a
--- /dev/null
+++ b/meta-openstack/recipes-devtools/python/python-novaclient/fix_novaclient_memory_leak.patch
@@ -0,0 +1,103 @@
+---
+ novaclient/client.py                            |   10 ++++++++++
+ novaclient/openstack/common/apiclient/client.py |   10 ++++++++++
+ novaclient/v1_1/client.py                       |    6 ++++++
+ novaclient/v3/client.py                         |    6 ++++++
+ 4 files changed, 32 insertions(+)
+
+--- a/novaclient/client.py
++++ b/novaclient/client.py
+@@ -87,6 +87,7 @@
+             self.timeout = None
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+ 
+         self.management_url = self.bypass_url or None
+         self.auth_token = auth_token
+@@ -139,6 +140,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def http_log_req(self, method, url, kwargs):
+         if not self.http_log_debug:
+             return
+@@ -214,6 +221,9 @@
+         resp, body = self.request(url, method, **kwargs)
+         self.times.append(("%s %s" % (method, url),
+                            start_time, time.time()))
++        # remove oldest items until we maintain max length
++        while len(self.times) > self.times_max_len:
++            del self.times[0]
+         return resp, body
+ 
+     def _cs_request(self, url, method, **kwargs):
+--- a/novaclient/openstack/common/apiclient/client.py
++++ b/novaclient/openstack/common/apiclient/client.py
+@@ -90,6 +90,7 @@
+         self.user_agent = user_agent or self.user_agent
+ 
+         self.times = []  # [("item", starttime, endtime), ...]
++        self.times_max_len = 200
+         self.timings = timings
+ 
+         # requests within the same session can reuse TCP connections from pool
+@@ -142,6 +143,12 @@
+     def reset_timings(self):
+         self.times = []
+ 
++    def get_timings_max_len(self):
++        return self.times_max_len
++
++    def set_timings_max_len(self, new_len):
++        self.times_max_len = new_len
++
+     def request(self, method, url, **kwargs):
+         """Send an http request with the specified characteristics.
+ 
+@@ -173,6 +180,9 @@
+         if self.timings:
+             self.times.append(("%s %s" % (method, url),
+                                start_time, time.time()))
++            # remove oldest items until we maintain max length
++            while len(self.times) > self.times_max_len:
++                del self.times[0]
+         self._http_log_resp(resp)
+ 
+         if resp.status_code >= 400:
+--- a/novaclient/v1_1/client.py
++++ b/novaclient/v1_1/client.py
+@@ -156,6 +156,12 @@
+     def reset_timings(self):
+         self.client.reset_timings()
+ 
++    def get_timings_max_len(self):
++        return self.client.get_timings_max_len()
++
++    def set_timings_max_len(self, new_len):
++        self.client.set_timings_max_len(new_len)
++
+     def authenticate(self):
+         """
+         Authenticate against the server.
+--- a/novaclient/v3/client.py
++++ b/novaclient/v3/client.py
+@@ -121,6 +121,12 @@
+     def reset_timings(self):
+         self.client.reset_timings()
+ 
++    def get_timings_max_len(self):
++        return self.client.get_timings_max_len()
++
++    def set_timings_max_len(self, new_len):
++        self.client.set_timings_max_len(new_len)
++
+     def authenticate(self):
+         """
+         Authenticate against the server.
diff --git a/meta-openstack/recipes-devtools/python/python-novaclient_git.bb b/meta-openstack/recipes-devtools/python/python-novaclient_git.bb
index 2884e0f..7ac9b34 100644
--- a/meta-openstack/recipes-devtools/python/python-novaclient_git.bb
+++ b/meta-openstack/recipes-devtools/python/python-novaclient_git.bb
@@ -7,7 +7,10 @@ DEPENDS = "python-setuptools-git"
 
 PR = "r0"
 
-SRC_URI = "git://github.com/openstack/python-novaclient.git;branch=master"
+SRC_URI = "\
+        git://github.com/openstack/python-novaclient.git;branch=master \
+        file://fix_novaclient_memory_leak.patch \
+        "
 
 PV="2.16.0+git${SRCPV}"
 SRCREV="64043442bbafa48f9042b669d30292b1db00db4f"