Merge branch '25-fix-issues-with-python-license-detection' into 'dev'

Resolve "fix issues with Python license detection"

See merge request software/services/fair/ci-services!20

.gitlab-ci.yml

@@ -9,7 +9,7 @@ stages:
   - ci-services
 
 variables:
-  VERSION: 0.4.1
+  VERSION: 0.4.2
   DOCKER_DST_IMAGE_NAME: ci-services
 
 build-deploy:dep:

build/docker/dev/Dockerfile

@@ -37,7 +37,7 @@ RUN apk upgrade
 # copy entrypoint script to executables
 COPY ./build/docker/dev/docker-entrypoint.sh /usr/local/bin/
 COPY ./build/docker/dev/entrypoint-manual.txt /usr/local/bin/
-RUN echo '0.4.1' > /version.txt
+RUN echo '0.4.2' > /version.txt
 
 # copy service scripts to root dir
 COPY ./src/services /services

meta/ci-services-report.md

@@ -9,11 +9,11 @@
 
 **Software Location:** [gitext](https://gitext.gfz-potsdam.de/software/services/fair/ci-services)
 
-**Last Commit:** b6dd47947df3853deced91a4b65c83e044166a0c
+**Last Commit:** 20ac892e7ef7ec51a2764d42e70290a4a49c19bf
 
-**Report Time:** 26/05/2020 07:50:59
+**Report Time:** 26/05/2020 10:15:06
 
-**Report Version:** 0.4.1
+**Report Version:** 0.4.2
 
 ---
 
@@ -36,9 +36,9 @@
 |---|---|
 |Dockerfile|6617|
 |HTML|1834|
-|Python|61126|
+|Python|63274|
 |Ruby|2990|
-|Shell|20264|
+|Shell|20265|
 
 ### list used licenses
 
@@ -66,8 +66,8 @@
 * Unused licenses:
 * Used licenses: CC-BY-4.0, CC0-1.0, GPL-3.0-or-later
 * Read errors: 0
-* Files with copyright information: 69 / 69
-* Files with license information: 69 / 69
+* Files with copyright information: 70 / 70
+* Files with license information: 70 / 70
 
 Congratulations! Your project is compliant with version 3.0 of the REUSE Specification :-)
 

src/services/list-used-licenses/functions/language_parser.py

@@ -53,7 +53,7 @@ def get_python_libs(file_path=''):
             while line:
                 if line[0] != '#' and line != '' and line[0:2] != '-r' and line != '\n':
                     stripped_down_line = line.strip().split('#')[0].replace(' ', '')
-                    if stripped_down_line[0:4] == 'http':
+                    if ('/' or ':') in stripped_down_line:
                         used_libs_without_stdlib_list.extend([{stripped_down_line: 'unknown'}])
                     elif '/' in stripped_down_line:
                         used_libs_without_stdlib_list.extend([{stripped_down_line: 'unknown'}])
@@ -79,10 +79,22 @@ def get_python_libs(file_path=''):
                 line = file.readline()
             used_libs_list = []
             for line in lines_with_libs_list:
+                lib_list = []
                 if line[0:7] == 'import ':
-                    used_libs_list = used_libs_list + line[7:].replace(' ', '').split(",")
+                    lib_list = line[7:].replace(' ', '').split(",")
                 if line[0:5] == 'from ':
-                    used_libs_list.append(line.split(' ')[1])
+                    lib_list = [line.split(' ')[1]]
+                tmp_lib_list =[]
+                for lib in lib_list:
+                    if lib is not None:
+                        if lib[0] == '.':
+                            tmp_lib = lib[1:]
+                            tmp_lib_list.append('.' + tmp_lib.split('.')[0])
+                        else:
+                            tmp_lib = lib
+                            tmp_lib_list.append(tmp_lib.split('.')[0])
+
+                used_libs_list += tmp_lib_list
 
         used_libs_list = list(dict.fromkeys(used_libs_list))
 

src/services/list-used-licenses/functions/license_requester.py

@@ -59,7 +59,7 @@ def get_python_libraries_license(libraries_dict):
             license_dict['licenses'][lib] = 'Python-2.0'
             continue
         # handle unknown location
-        if lib[0:4] == 'http' or '.' in lib or '/' in lib:
+        if '.' in lib or '/' in lib or ':' in lib:
             license_dict['unknowns'][lib] = 'unknown location'
             continue
         # handle unknown version

src/services/list-used-licenses/list-used-licenses.py

@@ -272,7 +272,8 @@ report = {'name': project_name,
           'report version': report_version,
           'list used licenses': {'description': 'Generates a list of dependencies and their license.',
                                  'result': {'license list': cumulated_licenses_dict,
-                                            'unknowns': unknowns}}}
+                                            'unknowns': unknowns},
+                                 'debug used libs': used_libs_dict}}
 
 # create report
 

test/generate_local_test_report.sh

@@ -36,7 +36,7 @@ tail -n +7 test/test_report.md > test/tmp_test_report.md
 IS_CHK_SUM=$(sha512sum meta/ci-services-report.md | awk '{ print $1 }')
 PASS_CHK_SUM=$(sha512sum test/tmp_test_report.md | awk '{ print $1 }')
 
-rm test/tmp_test_report.md
+#rm test/tmp_test_report.md
 
 if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then
   echo "Generated report does not equal the test template"

test/test_report.md

@@ -19,7 +19,7 @@ SPDX-License-Identifier: CC0-1.0
 
 **Report Time:** 0
 
-**Report Version:** 0.4.1
+**Report Version:** 0.4.2
 
 ---
 
@@ -56,8 +56,8 @@ SPDX-License-Identifier: CC0-1.0
 |---|---|---|
 |Apache 2.0|1|requests|
 |GPL-3.0-or-later AND Apache-2.0 AND CC0-1.0 AND CC-BY-SA-4.0|1|reuse|
-|MIT|2|PyYAML <br> stdlib-list|
-|Python-2.0|10|datetime <br> importlib.util <br> json <br> os <br> pathlib <br> re <br> shutil <br> sys <br> time <br> unittest|
+|MIT|3|PyYAML <br> httpretty <br> stdlib-list|
+|Python-2.0|10|datetime <br> importlib <br> json <br> os <br> pathlib <br> re <br> shutil <br> sys <br> time <br> unittest|
 |mock|1|stdlib|
 
 #### Unknown objects
@@ -78,6 +78,8 @@ SPDX-License-Identifier: CC0-1.0
 
 * Python / coverage / unsupported version modifier (!=3.5)
 
+* Python / django / unknown version
+
 * Python / docopt / unsupported version modifier (>=0.5.0)
 
 * Python / green / unknown version
@@ -105,8 +107,8 @@ SPDX-License-Identifier: CC0-1.0
 * Unused licenses:
 * Used licenses: CC-BY-4.0, CC0-1.0, GPL-3.0-or-later
 * Read errors: 0
-* Files with copyright information: 68 / 68
-* Files with license information: 68 / 68
+* Files with copyright information: 69 / 69
+* Files with license information: 69 / 69
 
 Congratulations! Your project is compliant with version 3.0 of the REUSE Specification :-)
 

test/testfiles/Python/test-python-imports.py

+# ci-services
+# This Software provides services to check and improve the source code quality of software-projects
+#
+# SPDX-FileCopyrightText: 2020 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences Potsdam, Germany (https://www.gfz-potsdam.de/)
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version, complemented with
+# the following provision:
+# For the scientific transparency and verification of results obtained
+# and communicated to the public after using a modified version of the
+# work, You (as the recipient of the source code and author of this
+# modified version, used to produce the published results in scientific
+# communications) commit to make this modified source code available in
+# a repository that is easily and freely accessible for a duration of
+# five years after the communication of the obtained results.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.contrib.auth.mixins import UserPassesTestMixin
+from django.shortcuts import redirect
+from django.urls import reverse
+import httpretty

test/testfiles/Python/test-requirements-dev.txt

@@ -24,4 +24,5 @@ http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a88
 #   Same as 1st section, just here to show that you can put things in any order.
 rejected
 green
+httpretty == 1.0.2
 #
\ No newline at end of file