From d93646cb8faa25d7b40035be0a48a5cb0af89081 Mon Sep 17 00:00:00 2001 From: Maximilian Dolling Date: Sun, 22 Mar 2020 15:23:45 +0100 Subject: [PATCH] Resolve "hook up ci-services" --- .gitlab-ci.yml | 170 ++++++++++++++---- README.md | 17 ++ build/dependencies/base/dependencies_OS.txt | 1 + build/docker/dev/entrypoint-manual.txt | 8 +- .../docker/{deploy => production}/Dockerfile | 8 +- .../docker-entrypoint.sh | 8 + .../entrypoint-manual.txt | 8 +- meta/ci-services-report.md | 29 +++ .../service_language_detection.rb | 3 + .../service_language_detection.sh | 1 - src/services/report/service_report.py | 9 +- src/services/test/service_test.py | 2 + test/test_report.md | 2 + 13 files changed, 220 insertions(+), 46 deletions(-) rename build/docker/{deploy => production}/Dockerfile (67%) rename build/docker/{deploy => production}/docker-entrypoint.sh (79%) rename build/docker/{deploy => production}/entrypoint-manual.txt (57%) create mode 100755 meta/ci-services-report.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07d2acd..ce94171 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,15 @@ stages: - - build-deploy-dep - - build-deploy-dev - - test + - build-deploy:dep - build-deploy + - test + - ci-services variables: - VERSION: 0.0.3 + VERSION: 0.0.4 DOCKER_DST_IMAGE_NAME: ci-services -build-dep-image: - stage: build-deploy-dep +build-deploy:dep: + stage: build-deploy:dep image: docker:19.03.1 tags: - ci-services @@ -24,15 +24,14 @@ build-dep-image: changes: - "build/dependencies/**/*" - "build/docker/dep/**/*" - - ".gitlab-ci.yml" -build-dev-image: - stage: build-deploy-dev +build-deploy:dev: + stage: build-deploy image: docker:19.03.1 tags: - ci-services dependencies: - - build-dep-image + - build-deploy:dep variables: DOCKER_FILE: build/docker/dev/Dockerfile before_script: @@ -41,16 +40,37 @@ build-dev-image: - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" except: - - master + refs: + - master only: changes: - - "build/dependencies/**/*" - - "build/docker/dep/**/*" - - ".gitlab-ci.yml" - - "build/docker/dev/**/*" + - "build/**/*" - "src/**/*" -test-dev-image: +build-deploy:production: + stage: build-deploy + image: docker:19.03.1 + tags: + - ci-services + dependencies: + - build-deploy:dep + variables: + DOCKER_FILE: build/docker/production/Dockerfile + before_script: + - echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + script: + - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} + - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" + - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} + - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" + only: + refs: + - master + changes: + - "build/**/*" + - "src/**/*" + +test:dev: stage: test image: name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" @@ -58,7 +78,7 @@ test-dev-image: tags: - ci-services dependencies: - - build-dev-image + - build-deploy:dev script: - mkdir -p /repo - cp -r "$CI_PROJECT_DIR/." /repo @@ -77,33 +97,121 @@ test-dev-image: changes: - "build/**/*" - "src/**/*" - - ".gitlab-ci.yml" + - "test/**/*" artifacts: paths: - meta - when: always + when: on_failure expire_in: 1 day -build-ci-service-image: - stage: build-deploy - image: docker:19.03.1 +test:production: + stage: test + image: + name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" + entrypoint: [""] tags: - ci-services dependencies: - - build-dep-image - variables: - DOCKER_FILE: build/docker/deploy/Dockerfile + - build-deploy:production + script: + - mkdir -p /repo + - cp -r "$CI_PROJECT_DIR/." /repo + - docker-entrypoint.sh --test + - cp -rf /repo/. "$CI_PROJECT_DIR" + - IS_CHK_SUM=$(sha512sum /repo/meta/ci-services-report.md | awk '{ print $1 }') + - PASS_CHK_SUM=$(sha512sum /repo/test/test_report.md | awk '{ print $1 }') + - > + if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then + echo "Generated report does not equal the test template" + exit 1 + fi + only: + refs: + - master + changes: + - "build/**/*" + - "src/**/*" + - "test/**/*" + artifacts: + paths: + - meta + when: on_failure + expire_in: 1 day + +ci-services:dev: + stage: ci-services + image: + name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" + entrypoint: [""] + tags: + - ci-services + dependencies: + - test:dev before_script: - - echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts script: - - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" - - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" + - mkdir -p /repo + - cp -r "$CI_PROJECT_DIR/." /repo + - docker-entrypoint.sh -a + - cp -rf /repo/meta "$CI_PROJECT_DIR" + - eval $(ssh-agent -s) + - echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add - + - git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}" + - git config user.email "ci-services@gitext.gfz-potsdam.de" + - git add -f meta/ci-services-report.md + - git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci" + - git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}" + - exit 0 + except: + - master + only: + changes: + - "build/**/*" + - "src/**/*" + - "test/**/*" + artifacts: + paths: + - meta/ci-services-report.md + expire_in: 1 day + +ci-services:production: + stage: ci-services + image: + name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" + entrypoint: [""] + tags: + - ci-services + dependencies: + - test:production + before_script: + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + script: + - mkdir -p /repo + - cp -r "$CI_PROJECT_DIR/." /repo + - docker-entrypoint.sh -a + - cp -rf /repo/meta "$CI_PROJECT_DIR" + - eval $(ssh-agent -s) + - echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add - + - git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}" + - git config user.email "ci-services@gitext.gfz-potsdam.de" + - git add -f meta/ci-services-report.md + - git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci" + - git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}" + - exit 0 only: refs: - master changes: - "build/**/*" - "src/**/*" - - ".gitlab-ci.yml" + - "test/**/*" + artifacts: + paths: + - meta/ci-services-report.md + expire_in: 1 day diff --git a/README.md b/README.md index 160061b..e23e5bc 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,20 @@ docker run -v [/absolute/path/to/repo]:/repo \ * curate meta data * generate report * test for code hosting + +# tmp + +**SSH_KNOWN_HOSTS_GITLAB** +# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3 +gitext.gfz-potsdam.de ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsiq5txhABcG9oePf47SCd+w/tfSs6NWKaMRd2BfBi9DGORQX4wwV4CYvSJbMfJvQO5SMGwhnEAXe0bWihC2V8X7lzcePrpjfP+uH2lmipcFbV9g3iQpM7Fusr96IV65v/qQ6HE4+KkHJLR1vEKj1AOaVgDww7CHhVAHCkvqcSwwtfOjBbqyfQ2Su7O6UNzs1ecQcIXnQNO8ebzimv3c8mKdo9j8i5eiWVqzRivmi/R3XIIc2T695mTLyxUplqfyvZpDL90F9VEjr/3HM9/3dASbonaTH8SoMRCxOy8Fpz0Va1MdpSor0uoemQjbu8aZzGB0+tsE5eWjuFaZib8QDF +# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3 +gitext.gfz-potsdam.de ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFiDZULQkGfyG/dFvTtoE3PoANi1PccdjuYfdRId1LaqDEE/sDdwJoy/Ate0DbOoTrjfb22eawUkjVVEmvInXh4= +# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3 +gitext.gfz-potsdam.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGRXeIQaIUcyF17/3TTZR2ILHoKqYpAD6/E3s8YTaq8G + + +**create deploy key** + +ssh-keygen -t rsa -b 4096 -C "ci-services" +public key > settings/repository/deploy keys (write access must be allowed) +private key > settings/CI/vars \ No newline at end of file diff --git a/build/dependencies/base/dependencies_OS.txt b/build/dependencies/base/dependencies_OS.txt index db99ba5..72905c8 100644 --- a/build/dependencies/base/dependencies_OS.txt +++ b/build/dependencies/base/dependencies_OS.txt @@ -1,2 +1,3 @@ git bash +openssh-client \ No newline at end of file diff --git a/build/docker/dev/entrypoint-manual.txt b/build/docker/dev/entrypoint-manual.txt index 991b59b..7a98e30 100644 --- a/build/docker/dev/entrypoint-manual.txt +++ b/build/docker/dev/entrypoint-manual.txt @@ -6,9 +6,11 @@ Runs various services for software quality detection. Needs a directory to run services on bound to '/repo' Available services: + -a, --all runs all services (reporting is last) -h, --help shows this help page + -l, --language lists used programming languages in given repo + -r, --report consolidates reports from previous ran services + generates markdown report -t, --test show the content of the repo to run services on and list all available services - and runs all available services with certain debug flags - -a, --all runs all services (reporting is last) - -l, --language lists used programming languages in given repo \ No newline at end of file + and runs all available services with certain debug flags \ No newline at end of file diff --git a/build/docker/deploy/Dockerfile b/build/docker/production/Dockerfile similarity index 67% rename from build/docker/deploy/Dockerfile rename to build/docker/production/Dockerfile index d43f768..9f471cc 100644 --- a/build/docker/deploy/Dockerfile +++ b/build/docker/production/Dockerfile @@ -7,14 +7,14 @@ RUN apk update RUN apk upgrade # copy entrypoint script to executables -COPY ./build/docker/deploy/docker-entrypoint.sh /usr/local/bin/ -COPY ./build/docker/deploy/entrypoint-manual.txt /usr/local/bin/ +COPY ./build/docker/production/docker-entrypoint.sh /usr/local/bin/ +COPY ./build/docker/production/entrypoint-manual.txt /usr/local/bin/ # copy service scripts to root dir COPY ./src/services /services -# cleanup -RUN rm -rf /build +# clean up +RUN rm -rf /services/test ENTRYPOINT ["docker-entrypoint.sh"] CMD ["--help"] diff --git a/build/docker/deploy/docker-entrypoint.sh b/build/docker/production/docker-entrypoint.sh similarity index 79% rename from build/docker/deploy/docker-entrypoint.sh rename to build/docker/production/docker-entrypoint.sh index b08eff8..33eae2c 100755 --- a/build/docker/deploy/docker-entrypoint.sh +++ b/build/docker/production/docker-entrypoint.sh @@ -23,8 +23,16 @@ then exit 0 fi +if [ "$1" = "--language" ] || [ "$1" = "-l" ] +then + /services/language_detection/service_language_detection.sh + chmod -R 777 /repo/meta + exit 0 +fi + if [ "$1" = "--all" ] || [ "$1" = "-a" ] then + /services/language_detection/service_language_detection.sh /services/report/service_report.sh "$2" chmod -R 777 /repo/meta exit 0 diff --git a/build/docker/deploy/entrypoint-manual.txt b/build/docker/production/entrypoint-manual.txt similarity index 57% rename from build/docker/deploy/entrypoint-manual.txt rename to build/docker/production/entrypoint-manual.txt index e49f418..3c631c1 100644 --- a/build/docker/deploy/entrypoint-manual.txt +++ b/build/docker/production/entrypoint-manual.txt @@ -6,8 +6,8 @@ Runs various services for software quality detection. Needs a directory to run services on bound to '/repo' Available services: - -h, --help shows this help page - -t, --test show the content of the repo to run services on - and list all available services - and runs all available services with certain debug flags -a, --all runs all services (reporting is last) + -h, --help shows this help page + -l, --language lists used programming languages in given repo + -r, --report consolidates reports from previous ran services + generates markdown report \ No newline at end of file diff --git a/meta/ci-services-report.md b/meta/ci-services-report.md new file mode 100755 index 0000000..77dfa82 --- /dev/null +++ b/meta/ci-services-report.md @@ -0,0 +1,29 @@ +# Software Quality Report +> **Note:** This report is automatically generated by [ci-services](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services) + +--- + +### Meta Data + +**Software Name:** ci-services + +**Software Location:** [gitext](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services) + +**Last Commit:** 5a47f78a6f329f8c9664657bf933bf6bd03c9ada + +**Report Time:** 22/03/2020 01:20:37 + +--- + +### Completed Tests + +**Programming Language Detection** + +> **Description: Detects used programming languages. Output values are bytes of code.** + +|name|result| +|---|---| +|Dockerfile|1745| +|Python|4626| +|Ruby|1350| +|Shell|2959| diff --git a/src/services/language_detection/service_language_detection.rb b/src/services/language_detection/service_language_detection.rb index 655d5a1..67c24e9 100644 --- a/src/services/language_detection/service_language_detection.rb +++ b/src/services/language_detection/service_language_detection.rb @@ -12,6 +12,9 @@ project_name = project_name[0, project_name.length - 5] project_location = `echo $CI_PROJECT_URL` project_location = project_location[0, project_location.length - 1] +if project_location == '' + project_location = 'local' +end last_commit_ref = `git rev-parse HEAD` last_commit_ref = last_commit_ref[0, 40] diff --git a/src/services/language_detection/service_language_detection.sh b/src/services/language_detection/service_language_detection.sh index fdb23d4..6e9e93d 100755 --- a/src/services/language_detection/service_language_detection.sh +++ b/src/services/language_detection/service_language_detection.sh @@ -5,4 +5,3 @@ WORKING_DIR="/meta/language_detection" cd "$REPO_DIR" || exit 1 ruby /services/language_detection/service_language_detection.rb "$REPO_DIR" "$WORKING_DIR" -# bundle exec bin/github-linguist --breakdown diff --git a/src/services/report/service_report.py b/src/services/report/service_report.py index 3ddbefe..21f7a98 100644 --- a/src/services/report/service_report.py +++ b/src/services/report/service_report.py @@ -72,8 +72,11 @@ with open(working_dir + '/ci-services-report.md', "a+") as md: # write meta data md.write("### Meta Data\n\n") md.write("**Software Name:** {}\n\n".format(report_dict_ordered['name'])) - md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:], - report_dict_ordered['location'])) + if 'https' in report_dict_ordered['location']: + md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:], + report_dict_ordered['location'])) + else: + md.write("**Software Location:** local\n\n") md.write("**Last Commit:** {}\n\n".format(report_dict_ordered['last commit hash'])) md.write("**Report Time:** {}\n\n".format(report_dict_ordered['report time'])) md.write('---\n\n') @@ -82,7 +85,7 @@ with open(working_dir + '/ci-services-report.md', "a+") as md: md.write("### Completed Tests\n") for subreport in report_dict_ordered: if isinstance(report_dict_ordered[subreport], dict): - md.write("**{}**\n\n".format(subreport)) + md.write("\n**{}**\n\n".format(subreport)) md.write("> **Description: {}**\n\n".format(report_dict_ordered[subreport]['description'])) # empty table header diff --git a/src/services/test/service_test.py b/src/services/test/service_test.py index b22308f..06827da 100644 --- a/src/services/test/service_test.py +++ b/src/services/test/service_test.py @@ -11,6 +11,8 @@ working_dir = repo_dir + sys.argv[2] project_name = os.popen('basename $(git remote get-url origin)').read()[:-5] last_commit_ref = os.popen('git rev-parse HEAD').read()[0:40] project_location = os.popen('echo $CI_PROJECT_URL').read()[:-1] +if project_location == '': + project_location = 'local' report_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") # generate report as dict diff --git a/test/test_report.md b/test/test_report.md index 679d6cd..173e151 100644 --- a/test/test_report.md +++ b/test/test_report.md @@ -16,6 +16,7 @@ --- ### Completed Tests + **Programming Language Detection** > **Description: Detects used programming languages. Output values are bytes of code.** @@ -26,6 +27,7 @@ |Python|0| |Ruby|0| |Shell|0| + **test** > **Description: used for debug output in test cases** -- GitLab