Add a new task checklicense and fix some bugs in distro_check.py

distro_check.py: Create a new function called create_log_file to reduce a lot of repeat code in distrodata.bbclass.
                 We needn't to create log file in function save_distro_check_result, because the log file has been generated in check_eventhandler.
                 Another bug is that we maybe access the /tmp/Meego-1.0 before we create this file.
                 Add a judge statement to decide whether we need to create this file firstly.
distrodata.bbclass: Add a new task checklicense to collect missing text license information.
                    This can help package-report system to know how many recipes are missing license text.

(From OE-Core rev: b41148cda9f0cc292b662a8473f26bc1ee0148f3)

Signed-off-by: Mei Lei <lei.mei@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 119 insertions, 85 deletions

diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
index 905dad7b9c..f24cff8dc7 100644
--- a/meta/classes/distrodata.bbclass
+++ b/meta/classes/distrodata.bbclass
@@ -4,19 +4,9 @@ addhandler distro_eventhandler
 python distro_eventhandler() {
 
     if bb.event.getName(e) == "BuildStarted":
-        """initialize log files."""
-        logpath = bb.data.getVar('LOG_DIR', e.data, 1)
-        bb.utils.mkdirhier(logpath)
-        logfile = os.path.join(logpath, "distrodata.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
-        if not os.path.exists(logfile):
-                slogfile = os.path.join(logpath, "distrodata.csv")
-                if os.path.exists(slogfile):
-                        os.remove(slogfile)
-                os.system("touch %s" % logfile)
-                os.symlink(logfile, slogfile)
-                bb.data.setVar('LOG_FILE', logfile, e.data)
-
-        lf = bb.utils.lockfile(logfile + ".lock")
+        import oe.distro_check as dc
+        logfile = dc.create_log_file(e.data, "distrodata.csv")
+        lf = bb.utils.lockfile("%s.lock" % logfile)
         f = open(logfile, "a")
         f.write("Package,Description,Owner,License,ChkSum,Status,VerMatch,Version,Upsteam,Non-Update,Reason,Recipe Status\n")
         f.close()
@@ -33,9 +23,9 @@ python do_distrodata_np() {
         bb.note("Package Name: %s" % pn)
 
         import oe.distro_check as dist_check
-        tmpdir = bb.data.getVar('TMPDIR', d, 1)
+        tmpdir = bb.data.getVar('TMPDIR', d, True)
         distro_check_dir = os.path.join(tmpdir, "distro_check")
-        datetime = bb.data.getVar('DATETIME', localdata, 1)
+        datetime = bb.data.getVar('DATETIME', localdata, True)
         dist_check.update_distro_data(distro_check_dir, datetime)
 
         if pn.find("-native") != -1:
@@ -111,15 +101,15 @@ python do_distrodata_np() {
 addtask distrodata
 do_distrodata[nostamp] = "1"
 python do_distrodata() {
-        logpath = bb.data.getVar('LOG_DIR', d, 1)
+        logpath = bb.data.getVar('LOG_DIR', d, True)
         bb.utils.mkdirhier(logpath)
         logfile = os.path.join(logpath, "distrodata.csv")
 
         import oe.distro_check as dist_check
         localdata = bb.data.createCopy(d)
-        tmpdir = bb.data.getVar('TMPDIR', d, 1)
+        tmpdir = bb.data.getVar('TMPDIR', d, True)
         distro_check_dir = os.path.join(tmpdir, "distro_check")
-        datetime = bb.data.getVar('DATETIME', localdata, 1)
+        datetime = bb.data.getVar('DATETIME', localdata, True)
         dist_check.update_distro_data(distro_check_dir, datetime)
 
         pn = bb.data.getVar("PN", d, True)
@@ -189,7 +179,7 @@ python do_distrodata() {
         # do the comparison
         result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata)
 
-        lf = bb.utils.lockfile(logfile + ".lock")
+        lf = bb.utils.lockfile("%s.lock" % logfile)
         f = open(logfile, "a")
         f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," % \
                   (pname, pdesc, maintainer, plicense, pchksum, hasrstatus, vermatch, pcurver, pupver, noupdate, noupdate_reason, rstatus))
@@ -211,19 +201,10 @@ do_distrodataall() {
 addhandler checkpkg_eventhandler
 python checkpkg_eventhandler() {
     if bb.event.getName(e) == "BuildStarted":
-        """initialize log files."""
-        logpath = bb.data.getVar('LOG_DIR', e.data, 1)
-        bb.utils.mkdirhier(logpath)
-        logfile = os.path.join(logpath, "checkpkg.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
-        if not os.path.exists(logfile):
-                slogfile = os.path.join(logpath, "checkpkg.csv")
-                if os.path.exists(slogfile):
-                        os.remove(slogfile)
-                os.system("touch %s" % logfile)
-                os.symlink(logfile, slogfile)
-                bb.data.setVar('LOG_FILE', logfile, e.data)
-
-        lf = bb.utils.lockfile(logfile + ".lock")
+        import oe.distro_check as dc
+        logfile = dc.create_log_file(e.data, "checkpkg.csv")
+
+        lf = bb.utils.lockfile("%s.lock" % logfile)
         f = open(logfile, "a")
         f.write("Package\tVersion\tUpver\tLicense\tSection\tHome\tRelease\tPriority\tDepends\tBugTracker\tPE\tDescription\tStatus\tTracking\tURI\tMAINTAINER\n")
         f.close()
@@ -304,7 +285,7 @@ python do_checkpkg() {
                 Clear internal url cache as it's a temporary check. Not doing so will have 
                 bitbake check url multiple times when looping through a single url
                 """
-                fn = bb.data.getVar('FILE', d, 1)
+                fn = bb.data.getVar('FILE', d, True)
                 bb.fetch2.urldata_cache[fn] = {}
 
                 """
@@ -335,7 +316,7 @@ python do_checkpkg() {
         Return new version if success, or else error in "Errxxxx" style
         """
         def check_new_dir(url, curver, d):
-                pn = bb.data.getVar('PN', d, 1)
+                pn = bb.data.getVar('PN', d, True)
                 f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-1-" % pn)
                 status = internal_fetch_wget(url, d, f)
                 fhtml = f.read()
@@ -394,7 +375,7 @@ python do_checkpkg() {
                 """possible to have no version in pkg name, such as spectrum-fw"""
                 if not re.search("\d+", curname):
                         return pcurver
-                pn = bb.data.getVar('PN', d, 1)
+                pn = bb.data.getVar('PN', d, True)
                 f = tempfile.NamedTemporaryFile(delete=False, prefix="%s-2-" % pn)
                 status = internal_fetch_wget(url, d, f)
                 fhtml = f.read()
@@ -437,35 +418,35 @@ python do_checkpkg() {
                 f.close()
                 """if host hasn't directory information, no need to save tmp file"""
                 if status != "ErrHostNoDir" and re.match("Err", status):
-                        logpath = bb.data.getVar('LOG_DIR', d, 1)
+                        logpath = bb.data.getVar('LOG_DIR', d, True)
                         os.system("cp %s %s/" % (f.name, logpath))
                 os.unlink(f.name)
                 return status
 
         """first check whether a uri is provided"""
-        src_uri = bb.data.getVar('SRC_URI', d, 1)
+        src_uri = bb.data.getVar('SRC_URI', d, True)
         if not src_uri:
                 return
 
         """initialize log files."""
-        logpath = bb.data.getVar('LOG_DIR', d, 1)
+        logpath = bb.data.getVar('LOG_DIR', d, True)
         bb.utils.mkdirhier(logpath)
         logfile = os.path.join(logpath, "checkpkg.csv")
 
         """generate package information from .bb file"""
-        pname = bb.data.getVar('PN', d, 1)
-        pdesc = bb.data.getVar('DESCRIPTION', d, 1)
-        pgrp = bb.data.getVar('SECTION', d, 1)
-        pversion = bb.data.getVar('PV', d, 1)
-        plicense = bb.data.getVar('LICENSE',d,1)
-        psection = bb.data.getVar('SECTION',d,1)
-        phome = bb.data.getVar('HOMEPAGE', d, 1)
-        prelease = bb.data.getVar('PR',d,1)
-        ppriority = bb.data.getVar('PRIORITY',d,1)
-        pdepends = bb.data.getVar('DEPENDS',d,1)
-        pbugtracker = bb.data.getVar('BUGTRACKER',d,1)
-        ppe = bb.data.getVar('PE',d,1)
-        psrcuri = bb.data.getVar('SRC_URI',d,1)
+        pname = bb.data.getVar('PN', d, True)
+        pdesc = bb.data.getVar('DESCRIPTION', d, True)
+        pgrp = bb.data.getVar('SECTION', d, True)
+        pversion = bb.data.getVar('PV', d, True)
+        plicense = bb.data.getVar('LICENSE', d, True)
+        psection = bb.data.getVar('SECTION', d, True)
+        phome = bb.data.getVar('HOMEPAGE', d, True)
+        prelease = bb.data.getVar('PR', d, True)
+        ppriority = bb.data.getVar('PRIORITY', d, True)
+        pdepends = bb.data.getVar('DEPENDS', d, True)
+        pbugtracker = bb.data.getVar('BUGTRACKER', d, True)
+        ppe = bb.data.getVar('PE', d, True)
+        psrcuri = bb.data.getVar('SRC_URI', d, True)
 
         found = 0
         for uri in src_uri.split():
@@ -483,9 +464,9 @@ python do_checkpkg() {
 
         (type, host, path, user, pswd, parm) = bb.decodeurl(uri)
         if type in ['http', 'https', 'ftp']:
-                pcurver = bb.data.getVar('PV', d, 1)
+                pcurver = bb.data.getVar('PV', d, True)
         else:
-                pcurver = bb.data.getVar("SRCREV", d, 1)
+                pcurver = bb.data.getVar("SRCREV", d, True)
 
         if type in ['http', 'https', 'ftp']:
                 newver = pcurver
@@ -509,7 +490,7 @@ python do_checkpkg() {
                         newver = check_new_dir(alturi, dirver, d)
                         altpath = path
                         if not re.match("Err", newver) and dirver != newver:
-                                altpath = altpath.replace(dirver, newver, 1)
+                                altpath = altpath.replace(dirver, newver, True)
                                 
                 """Now try to acquire all remote files in current directory"""
                 if not re.match("Err", newver):
@@ -625,7 +606,7 @@ python do_checkpkg() {
                 pstatus += ":%s%s" % (host, path)
 
         """Read from manual distro tracking fields as alternative"""
-        pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, 1)
+        pmver = bb.data.getVar("RECIPE_LATEST_VERSION", d, True)
         if not pmver:
                 pmver = "N/A"
                 pmstatus = "ErrNoRecipeData"
@@ -639,7 +620,7 @@ python do_checkpkg() {
         psrcuri = psrcuri.split()[0]
         pdepends = "".join(pdepends.split("\t"))
         pdesc = "".join(pdesc.split("\t"))
-        lf = bb.utils.lockfile(logfile + ".lock")
+        lf = bb.utils.lockfile("%s.lock" % logfile)
         f = open(logfile, "a")
         f.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % \
                   (pname,pversion,pupver,plicense,psection, phome,prelease, ppriority,pdepends,pbugtracker,ppe,pdesc,pstatus,pmver,psrcuri,maintainer))
@@ -654,25 +635,12 @@ do_checkpkgall() {
         :
 }
 
-#addhandler check_eventhandler
-python check_eventhandler() {
+addhandler distro_check_eventhandler
+python distro_check_eventhandler() {
     if bb.event.getName(e) == "BuildStarted":
-        import oe.distro_check as dc
-        tmpdir = bb.data.getVar('TMPDIR', e.data, 1)
-        distro_check_dir = os.path.join(tmpdir, "distro_check")
-        datetime = bb.data.getVar('DATETIME', e.data, 1)
         """initialize log files."""
-        logpath = bb.data.getVar('LOG_DIR', e.data, 1)
-        bb.utils.mkdirhier(logpath)
-        logfile = os.path.join(logpath, "distrocheck.%s.csv" % bb.data.getVar('DATETIME', e.data, 1))
-        if not os.path.exists(logfile):
-                slogfile = os.path.join(logpath, "distrocheck.csv")
-                if os.path.exists(slogfile):
-                        os.remove(slogfile)
-                os.system("touch %s" % logfile)
-                os.symlink(logfile, slogfile)
-                bb.data.setVar('LOG_FILE', logfile, e.data)
-
+        import oe.distro_check as dc
+        result_file = dc.create_log_file(e.data, "distrocheck.csv")
     return
 }
 
@@ -681,18 +649,23 @@ do_distro_check[nostamp] = "1"
 python do_distro_check() {
     """checks if the package is present in other public Linux distros"""
     import oe.distro_check as dc
+    import bb
+    import shutil
     localdata = bb.data.createCopy(d)
     bb.data.update_data(localdata)
-    tmpdir = bb.data.getVar('TMPDIR', d, 1)
+    tmpdir = bb.data.getVar('TMPDIR', d, True)
     distro_check_dir = os.path.join(tmpdir, "distro_check")
-    datetime = bb.data.getVar('DATETIME', localdata, 1)
+    logpath = bb.data.getVar('LOG_DIR', d, True)
+    bb.utils.mkdirhier(logpath)
+    result_file = os.path.join(logpath, "distrocheck.csv")
+    datetime = bb.data.getVar('DATETIME', localdata, True)
     dc.update_distro_data(distro_check_dir, datetime)
 
     # do the comparison
     result = dc.compare_in_distro_packages_list(distro_check_dir, d)
 
     # save the results
-    dc.save_distro_check_result(result, datetime, d)
+    dc.save_distro_check_result(result, datetime, result_file, d)
 }
 
 addtask distro_checkall after do_distro_check
@@ -701,3 +674,55 @@ do_distro_checkall[nostamp] = "1"
 do_distro_checkall() {
         :
 }
+#
+#Check Missing License Text.
+#Use this task to generate the missing license text data for pkg-report system,
+#then we can search those recipes which license text isn't exsit in common-licenses directory
+#
+addhandler checklicense_eventhandler
+python checklicense_eventhandler() {
+    if bb.event.getName(e) == "BuildStarted":
+        """initialize log files."""
+        import oe.distro_check as dc
+        logfile = dc.create_log_file(e.data, "missinglicense.csv")
+        lf = bb.utils.lockfile("%s.lock" % logfile)
+        f = open(logfile, "a")
+        f.write("Package\tLicense\tMissingLicense\n")
+        f.close()
+        bb.utils.unlockfile(lf)
+    return
+}
+
+addtask checklicense
+do_checklicense[nostamp] = "1"
+python do_checklicense() {
+    import os
+    import bb
+    import shutil
+    logpath = bb.data.getVar('LOG_DIR', d, True)
+    bb.utils.mkdirhier(logpath)
+    pn = bb.data.getVar('PN', d, True)
+    logfile = os.path.join(logpath, "missinglicense.csv")
+    generic_directory = bb.data.getVar('COMMON_LICENSE_DIR', d, True)
+    license_types = bb.data.getVar('LICENSE', d, True)
+    for license_type in ((license_types.replace('+', '').replace('|', '&')
+                          .replace('(', '').replace(')', '').replace(';', '')
+                          .replace(',', '').replace(" ", "").split("&"))):
+        if not os.path.isfile(os.path.join(generic_directory, license_type)):
+            lf = bb.utils.lockfile("%s.lock" % logfile)
+            f = open(logfile, "a")
+            f.write("%s\t%s\t%s\n" % \
+                (pn,license_types,license_type))
+            f.close()
+            bb.utils.unlockfile(lf)
+    return
+}
+
+addtask checklicenseall after do_checklicense
+do_checklicenseall[recrdeptask] = "do_checklicense"
+do_checklicenseall[nostamp] = "1"
+do_checklicenseall() {
+        :
+}
+
+
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
index c85d4fb28b..55cdcad461 100644
--- a/meta/lib/oe/distro_check.py
+++ b/meta/lib/oe/distro_check.py
@@ -73,7 +73,8 @@ def clean_package_list(package_list):
 def get_latest_released_meego_source_package_list():
     "Returns list of all the name os packages in the latest meego distro"
 
-    
+    if not os.path.isfile("/tmp/Meego-1.0"):
+        os.mknod("/tmp/Meego-1.0")
     f = open("/tmp/Meego-1.0", "r")
     package_names = []
     for line in f:
@@ -341,7 +342,22 @@ def compare_in_distro_packages_list(distro_check_dir, d):
     bb.note("Matching: %s" % matching_distros)
     return matching_distros
 
-def save_distro_check_result(result, datetime, d):
+def create_log_file(d, logname):
+    logpath = bb.data.getVar('LOG_DIR', d, True)
+    bb.utils.mkdirhier(logpath)
+    logfn, logsuffix = os.path.splitext(logname)
+    logfile = os.path.join(logpath, "%s.%s%s" % (logfn, bb.data.getVar('DATETIME', d, True), logsuffix))
+    if not os.path.exists(logfile):
+            slogfile = os.path.join(logpath, logname)
+            if os.path.exists(slogfile):
+                    os.remove(slogfile)
+            os.system("touch %s" % logfile)
+            os.symlink(logfile, slogfile)
+            bb.data.setVar('LOG_FILE', logfile, d)
+    return logfile
+
+
+def save_distro_check_result(result, datetime, result_file, d):
     pn = bb.data.getVar('PN', d, True)
     logdir = bb.data.getVar('LOG_DIR', d, True)
     if not logdir:
@@ -349,16 +365,9 @@ def save_distro_check_result(result, datetime, d):
         return
     if not os.path.isdir(logdir):
         os.makedirs(logdir)
-    result_file = os.path.join(logdir, "distrocheck.%s.csv" % datetime)
     line = pn
     for i in result:
         line = line + "," + i
-    if not os.path.exists(result_file):
-        sresult_file = os.path.join(logdir, "distrocheck.csv")
-        if os.path.exists(sresult_file):
-            os.remove(sresult_file)
-        os.system("touch %s" % result_file)
-        os.symlink(result_file, sresult_file)
     f = open(result_file, "a")
     import fcntl
     fcntl.lockf(f, fcntl.LOCK_EX)