Resolve "hook up ci-services"

.gitlab-ci.yml

 stages:
-  - build-deploy-dep
-  - build-deploy-dev
-  - test
+  - build-deploy:dep
   - build-deploy
+  - test
+  - ci-services
 
 variables:
-  VERSION: 0.0.3
+  VERSION: 0.0.4
   DOCKER_DST_IMAGE_NAME: ci-services
 
-build-dep-image:
-  stage: build-deploy-dep
+build-deploy:dep:
+  stage: build-deploy:dep
   image: docker:19.03.1
   tags:
     - ci-services
@@ -24,15 +24,14 @@ build-dep-image:
     changes:
       - "build/dependencies/**/*"
       - "build/docker/dep/**/*"
-      - ".gitlab-ci.yml"
 
-build-dev-image:
-  stage: build-deploy-dev
+build-deploy:dev:
+  stage: build-deploy
   image: docker:19.03.1
   tags:
     - ci-services
   dependencies:
-    - build-dep-image
+    - build-deploy:dep
   variables:
     DOCKER_FILE: build/docker/dev/Dockerfile
   before_script:
@@ -41,16 +40,37 @@ build-dev-image:
     - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
     - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
   except:
-    - master
+    refs:
+      - master
   only:
     changes:
-      - "build/dependencies/**/*"
-      - "build/docker/dep/**/*"
-      - ".gitlab-ci.yml"
-      - "build/docker/dev/**/*"
+      - "build/**/*"
       - "src/**/*"
 
-test-dev-image:
+build-deploy:production:
+  stage: build-deploy
+  image: docker:19.03.1
+  tags:
+    - ci-services
+  dependencies:
+    - build-deploy:dep
+  variables:
+    DOCKER_FILE: build/docker/production/Dockerfile
+  before_script:
+    - echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
+  script:
+    - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
+    - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION"
+    - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
+    - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
+  only:
+    refs:
+      - master
+    changes:
+      - "build/**/*"
+      - "src/**/*"
+
+test:dev:
   stage: test
   image:
     name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
@@ -58,7 +78,7 @@ test-dev-image:
   tags:
     - ci-services
   dependencies:
-    - build-dev-image
+    - build-deploy:dev
   script:
     - mkdir -p  /repo
     - cp -r "$CI_PROJECT_DIR/." /repo
@@ -77,33 +97,121 @@ test-dev-image:
     changes:
       - "build/**/*"
       - "src/**/*"
-      - ".gitlab-ci.yml"
+      - "test/**/*"
   artifacts:
     paths:
       - meta
-    when: always
+    when: on_failure
     expire_in: 1 day
 
-build-ci-service-image:
-  stage: build-deploy
-  image: docker:19.03.1
+test:production:
+  stage: test
+  image:
+    name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
+    entrypoint: [""]
   tags:
     - ci-services
   dependencies:
-    - build-dep-image
-  variables:
-    DOCKER_FILE: build/docker/deploy/Dockerfile
+    - build-deploy:production
+  script:
+    - mkdir -p  /repo
+    - cp -r "$CI_PROJECT_DIR/." /repo
+    - docker-entrypoint.sh --test
+    - cp -rf /repo/. "$CI_PROJECT_DIR"
+    - IS_CHK_SUM=$(sha512sum /repo/meta/ci-services-report.md | awk '{ print $1 }')
+    - PASS_CHK_SUM=$(sha512sum /repo/test/test_report.md | awk '{ print $1 }')
+    - >
+      if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then
+        echo "Generated report does not equal the test template"
+        exit 1
+      fi
+  only:
+    refs:
+      - master
+    changes:
+      - "build/**/*"
+      - "src/**/*"
+      - "test/**/*"
+  artifacts:
+    paths:
+      - meta
+    when: on_failure
+    expire_in: 1 day
+
+ci-services:dev:
+  stage: ci-services
+  image:
+    name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
+    entrypoint: [""]
+  tags:
+    - ci-services
+  dependencies:
+    - test:dev
   before_script:
-    - echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
   script:
-    - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
-    - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION"
-    - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
-    - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
+    - mkdir -p  /repo
+    - cp -r "$CI_PROJECT_DIR/." /repo
+    - docker-entrypoint.sh -a
+    - cp -rf /repo/meta "$CI_PROJECT_DIR"
+    - eval $(ssh-agent -s)
+    - echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
+    - git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
+    - git config user.email "ci-services@gitext.gfz-potsdam.de"
+    - git add -f meta/ci-services-report.md
+    - git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
+    - git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
+    - exit 0
+  except:
+    - master
+  only:
+    changes:
+      - "build/**/*"
+      - "src/**/*"
+      - "test/**/*"
+  artifacts:
+    paths:
+      - meta/ci-services-report.md
+    expire_in: 1 day
+
+ci-services:production:
+  stage: ci-services
+  image:
+    name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
+    entrypoint: [""]
+  tags:
+    - ci-services
+  dependencies:
+    - test:production
+  before_script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - mkdir -p  /repo
+    - cp -r "$CI_PROJECT_DIR/." /repo
+    - docker-entrypoint.sh -a
+    - cp -rf /repo/meta "$CI_PROJECT_DIR"
+    - eval $(ssh-agent -s)
+    - echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
+    - git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
+    - git config user.email "ci-services@gitext.gfz-potsdam.de"
+    - git add -f meta/ci-services-report.md
+    - git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
+    - git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
+    - exit 0
   only:
     refs:
       - master
     changes:
       - "build/**/*"
       - "src/**/*"
-      - ".gitlab-ci.yml"
+      - "test/**/*"
+  artifacts:
+    paths:
+      - meta/ci-services-report.md
+    expire_in: 1 day

README.md

@@ -34,3 +34,20 @@ docker run -v [/absolute/path/to/repo]:/repo \
 * curate meta data
 * generate report
 * test for code hosting
+
+# tmp
+
+**SSH_KNOWN_HOSTS_GITLAB**
+# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
+gitext.gfz-potsdam.de ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsiq5txhABcG9oePf47SCd+w/tfSs6NWKaMRd2BfBi9DGORQX4wwV4CYvSJbMfJvQO5SMGwhnEAXe0bWihC2V8X7lzcePrpjfP+uH2lmipcFbV9g3iQpM7Fusr96IV65v/qQ6HE4+KkHJLR1vEKj1AOaVgDww7CHhVAHCkvqcSwwtfOjBbqyfQ2Su7O6UNzs1ecQcIXnQNO8ebzimv3c8mKdo9j8i5eiWVqzRivmi/R3XIIc2T695mTLyxUplqfyvZpDL90F9VEjr/3HM9/3dASbonaTH8SoMRCxOy8Fpz0Va1MdpSor0uoemQjbu8aZzGB0+tsE5eWjuFaZib8QDF
+# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
+gitext.gfz-potsdam.de ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFiDZULQkGfyG/dFvTtoE3PoANi1PccdjuYfdRId1LaqDEE/sDdwJoy/Ate0DbOoTrjfb22eawUkjVVEmvInXh4=
+# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
+gitext.gfz-potsdam.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGRXeIQaIUcyF17/3TTZR2ILHoKqYpAD6/E3s8YTaq8G
+
+
+**create deploy key**
+
+ssh-keygen -t rsa -b 4096 -C "ci-services"
+public key > settings/repository/deploy keys (write access must be allowed)
+private key > settings/CI/vars
\ No newline at end of file

build/dependencies/base/dependencies_OS.txt

 git
 bash
+openssh-client
\ No newline at end of file

build/docker/dev/entrypoint-manual.txt

@@ -6,9 +6,11 @@ Runs various services for software quality detection.
 Needs a directory to run services on bound to '/repo'
 
 Available services:
+  -a, --all                   runs all services (reporting is last)
   -h, --help                  shows this help page
+  -l, --language              lists used programming languages in given repo
+  -r, --report                consolidates reports from previous ran services
+                              generates markdown report
   -t, --test                  show the content of the repo to run services on
                               and list all available services
-                              and runs all available services with certain debug flags
-  -a, --all                   runs all services (reporting is last)
+                              and runs all available services with certain debug flags
\ No newline at end of file
-  -l, --language              lists used programming languages in given repo
\ No newline at end of file

build/docker/deploy/Dockerfile → build/docker/production/Dockerfile

@@ -7,14 +7,14 @@ RUN apk update
 RUN apk upgrade
 
 # copy entrypoint script to executables
-COPY ./build/docker/deploy/docker-entrypoint.sh /usr/local/bin/
-COPY ./build/docker/deploy/entrypoint-manual.txt /usr/local/bin/
+COPY ./build/docker/production/docker-entrypoint.sh /usr/local/bin/
+COPY ./build/docker/production/entrypoint-manual.txt /usr/local/bin/
 
 # copy service scripts to root dir
 COPY ./src/services /services
 
-# cleanup
-RUN rm -rf /build
+# clean up
+RUN rm -rf /services/test
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD ["--help"]

build/docker/deploy/docker-entrypoint.sh → build/docker/production/docker-entrypoint.sh

@@ -23,8 +23,16 @@ then
   exit 0
 fi
 
+if [ "$1" = "--language" ] || [ "$1" = "-l" ]
+then
+  /services/language_detection/service_language_detection.sh
+  chmod -R 777 /repo/meta
+  exit 0
+fi
+
 if [ "$1" = "--all" ] || [ "$1" = "-a" ]
 then
+  /services/language_detection/service_language_detection.sh
   /services/report/service_report.sh "$2"
   chmod -R 777 /repo/meta
   exit 0

build/docker/deploy/entrypoint-manual.txt → build/docker/production/entrypoint-manual.txt

@@ -6,8 +6,8 @@ Runs various services for software quality detection.
 Needs a directory to run services on bound to '/repo'
 
 Available services:
-  -h, --help                  shows this help page
-  -t, --test                  show the content of the repo to run services on
-                              and list all available services
-                              and runs all available services with certain debug flags
   -a, --all                   runs all services (reporting is last)
+  -h, --help                  shows this help page
+  -l, --language              lists used programming languages in given repo
+  -r, --report                consolidates reports from previous ran services
+                              generates markdown report
\ No newline at end of file

meta/ci-services-report.md

+# Software Quality Report
+> **Note:** This report is automatically generated by [ci-services](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services)
+
+---
+
+### Meta Data
+
+**Software Name:** ci-services
+
+**Software Location:** [gitext](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services)
+
+**Last Commit:** 5a47f78a6f329f8c9664657bf933bf6bd03c9ada
+
+**Report Time:** 22/03/2020 01:20:37
+
+---
+
+### Completed Tests
+
+**Programming Language Detection**
+
+> **Description: Detects used programming languages. Output values are bytes of code.**
+
+|name|result|
+|---|---|
+|Dockerfile|1745|
+|Python|4626|
+|Ruby|1350|
+|Shell|2959|

src/services/language_detection/service_language_detection.rb

@@ -12,6 +12,9 @@ project_name = project_name[0, project_name.length - 5]
 
 project_location = `echo $CI_PROJECT_URL`
 project_location = project_location[0, project_location.length - 1]
+if project_location == ''
+    project_location = 'local'
+end
 
 last_commit_ref = `git rev-parse HEAD`
 last_commit_ref = last_commit_ref[0, 40]

src/services/language_detection/service_language_detection.sh

@@ -5,4 +5,3 @@ WORKING_DIR="/meta/language_detection"
 
 cd "$REPO_DIR" || exit 1
 ruby /services/language_detection/service_language_detection.rb "$REPO_DIR" "$WORKING_DIR"
-# bundle exec bin/github-linguist --breakdown

src/services/report/service_report.py

@@ -72,8 +72,11 @@ with open(working_dir + '/ci-services-report.md', "a+") as md:
     # write meta data
     md.write("### Meta Data\n\n")
     md.write("**Software Name:** {}\n\n".format(report_dict_ordered['name']))
-    md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:],
-                                                          report_dict_ordered['location']))
+    if 'https' in report_dict_ordered['location']:
+        md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:],
+                                                              report_dict_ordered['location']))
+    else:
+        md.write("**Software Location:** local\n\n")
     md.write("**Last Commit:** {}\n\n".format(report_dict_ordered['last commit hash']))
     md.write("**Report Time:** {}\n\n".format(report_dict_ordered['report time']))
     md.write('---\n\n')
@@ -82,7 +85,7 @@ with open(working_dir + '/ci-services-report.md', "a+") as md:
     md.write("### Completed Tests\n")
     for subreport in report_dict_ordered:
         if isinstance(report_dict_ordered[subreport], dict):
-            md.write("**{}**\n\n".format(subreport))
+            md.write("\n**{}**\n\n".format(subreport))
             md.write("> **Description: {}**\n\n".format(report_dict_ordered[subreport]['description']))
 
             # empty table header

src/services/test/service_test.py

@@ -11,6 +11,8 @@ working_dir = repo_dir + sys.argv[2]
 project_name = os.popen('basename $(git remote get-url origin)').read()[:-5]
 last_commit_ref = os.popen('git rev-parse HEAD').read()[0:40]
 project_location = os.popen('echo $CI_PROJECT_URL').read()[:-1]
+if project_location == '':
+    project_location = 'local'
 report_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
 
 # generate report as dict

test/test_report.md

@@ -16,6 +16,7 @@
 ---
 
 ### Completed Tests
+
 **Programming Language Detection**
 
 > **Description: Detects used programming languages. Output values are bytes of code.**
@@ -26,6 +27,7 @@
 |Python|0|
 |Ruby|0|
 |Shell|0|
+
 **test**
 
 > **Description: used for debug output in test cases**