Commit d93646cb authored by Maximilian Dolling's avatar Maximilian Dolling

Resolve "hook up ci-services"

parent a48cd1b3
stages: stages:
- build-deploy-dep - build-deploy:dep
- build-deploy-dev
- test
- build-deploy - build-deploy
- test
- ci-services
variables: variables:
VERSION: 0.0.3 VERSION: 0.0.4
DOCKER_DST_IMAGE_NAME: ci-services DOCKER_DST_IMAGE_NAME: ci-services
build-dep-image: build-deploy:dep:
stage: build-deploy-dep stage: build-deploy:dep
image: docker:19.03.1 image: docker:19.03.1
tags: tags:
- ci-services - ci-services
...@@ -24,15 +24,14 @@ build-dep-image: ...@@ -24,15 +24,14 @@ build-dep-image:
changes: changes:
- "build/dependencies/**/*" - "build/dependencies/**/*"
- "build/docker/dep/**/*" - "build/docker/dep/**/*"
- ".gitlab-ci.yml"
build-dev-image: build-deploy:dev:
stage: build-deploy-dev stage: build-deploy
image: docker:19.03.1 image: docker:19.03.1
tags: tags:
- ci-services - ci-services
dependencies: dependencies:
- build-dep-image - build-deploy:dep
variables: variables:
DOCKER_FILE: build/docker/dev/Dockerfile DOCKER_FILE: build/docker/dev/Dockerfile
before_script: before_script:
...@@ -41,16 +40,37 @@ build-dev-image: ...@@ -41,16 +40,37 @@ build-dev-image:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" - docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
except: except:
- master refs:
- master
only: only:
changes: changes:
- "build/dependencies/**/*" - "build/**/*"
- "build/docker/dep/**/*"
- ".gitlab-ci.yml"
- "build/docker/dev/**/*"
- "src/**/*" - "src/**/*"
test-dev-image: build-deploy:production:
stage: build-deploy
image: docker:19.03.1
tags:
- ci-services
dependencies:
- build-deploy:dep
variables:
DOCKER_FILE: build/docker/production/Dockerfile
before_script:
- echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
script:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION"
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
only:
refs:
- master
changes:
- "build/**/*"
- "src/**/*"
test:dev:
stage: test stage: test
image: image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
...@@ -58,7 +78,7 @@ test-dev-image: ...@@ -58,7 +78,7 @@ test-dev-image:
tags: tags:
- ci-services - ci-services
dependencies: dependencies:
- build-dev-image - build-deploy:dev
script: script:
- mkdir -p /repo - mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo - cp -r "$CI_PROJECT_DIR/." /repo
...@@ -77,33 +97,121 @@ test-dev-image: ...@@ -77,33 +97,121 @@ test-dev-image:
changes: changes:
- "build/**/*" - "build/**/*"
- "src/**/*" - "src/**/*"
- ".gitlab-ci.yml" - "test/**/*"
artifacts: artifacts:
paths: paths:
- meta - meta
when: always when: on_failure
expire_in: 1 day expire_in: 1 day
build-ci-service-image: test:production:
stage: build-deploy stage: test
image: docker:19.03.1 image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
entrypoint: [""]
tags: tags:
- ci-services - ci-services
dependencies: dependencies:
- build-dep-image - build-deploy:production
variables: script:
DOCKER_FILE: build/docker/deploy/Dockerfile - mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh --test
- cp -rf /repo/. "$CI_PROJECT_DIR"
- IS_CHK_SUM=$(sha512sum /repo/meta/ci-services-report.md | awk '{ print $1 }')
- PASS_CHK_SUM=$(sha512sum /repo/test/test_report.md | awk '{ print $1 }')
- >
if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then
echo "Generated report does not equal the test template"
exit 1
fi
only:
refs:
- master
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta
when: on_failure
expire_in: 1 day
ci-services:dev:
stage: ci-services
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
entrypoint: [""]
tags:
- ci-services
dependencies:
- test:dev
before_script: before_script:
- echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" - mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script: script:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - mkdir -p /repo
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" - cp -r "$CI_PROJECT_DIR/." /repo
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.} - docker-entrypoint.sh -a
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" - cp -rf /repo/meta "$CI_PROJECT_DIR"
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
- git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
- git config user.email "ci-services@gitext.gfz-potsdam.de"
- git add -f meta/ci-services-report.md
- git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
- git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
- exit 0
except:
- master
only:
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta/ci-services-report.md
expire_in: 1 day
ci-services:production:
stage: ci-services
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
entrypoint: [""]
tags:
- ci-services
dependencies:
- test:production
before_script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh -a
- cp -rf /repo/meta "$CI_PROJECT_DIR"
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
- git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
- git config user.email "ci-services@gitext.gfz-potsdam.de"
- git add -f meta/ci-services-report.md
- git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
- git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
- exit 0
only: only:
refs: refs:
- master - master
changes: changes:
- "build/**/*" - "build/**/*"
- "src/**/*" - "src/**/*"
- ".gitlab-ci.yml" - "test/**/*"
artifacts:
paths:
- meta/ci-services-report.md
expire_in: 1 day
...@@ -34,3 +34,20 @@ docker run -v [/absolute/path/to/repo]:/repo \ ...@@ -34,3 +34,20 @@ docker run -v [/absolute/path/to/repo]:/repo \
* curate meta data * curate meta data
* generate report * generate report
* test for code hosting * test for code hosting
# tmp
**SSH_KNOWN_HOSTS_GITLAB**
# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
gitext.gfz-potsdam.de ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsiq5txhABcG9oePf47SCd+w/tfSs6NWKaMRd2BfBi9DGORQX4wwV4CYvSJbMfJvQO5SMGwhnEAXe0bWihC2V8X7lzcePrpjfP+uH2lmipcFbV9g3iQpM7Fusr96IV65v/qQ6HE4+KkHJLR1vEKj1AOaVgDww7CHhVAHCkvqcSwwtfOjBbqyfQ2Su7O6UNzs1ecQcIXnQNO8ebzimv3c8mKdo9j8i5eiWVqzRivmi/R3XIIc2T695mTLyxUplqfyvZpDL90F9VEjr/3HM9/3dASbonaTH8SoMRCxOy8Fpz0Va1MdpSor0uoemQjbu8aZzGB0+tsE5eWjuFaZib8QDF
# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
gitext.gfz-potsdam.de ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFiDZULQkGfyG/dFvTtoE3PoANi1PccdjuYfdRId1LaqDEE/sDdwJoy/Ate0DbOoTrjfb22eawUkjVVEmvInXh4=
# gitext.gfz-potsdam.de:22 SSH-2.0-OpenSSH_7.6p1 Ubuntu-4ubuntu0.3
gitext.gfz-potsdam.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGRXeIQaIUcyF17/3TTZR2ILHoKqYpAD6/E3s8YTaq8G
**create deploy key**
ssh-keygen -t rsa -b 4096 -C "ci-services"
public key > settings/repository/deploy keys (write access must be allowed)
private key > settings/CI/vars
\ No newline at end of file
git git
bash bash
openssh-client
\ No newline at end of file
...@@ -6,9 +6,11 @@ Runs various services for software quality detection. ...@@ -6,9 +6,11 @@ Runs various services for software quality detection.
Needs a directory to run services on bound to '/repo' Needs a directory to run services on bound to '/repo'
Available services: Available services:
-a, --all runs all services (reporting is last)
-h, --help shows this help page -h, --help shows this help page
-l, --language lists used programming languages in given repo
-r, --report consolidates reports from previous ran services
generates markdown report
-t, --test show the content of the repo to run services on -t, --test show the content of the repo to run services on
and list all available services and list all available services
and runs all available services with certain debug flags and runs all available services with certain debug flags
-a, --all runs all services (reporting is last) \ No newline at end of file
-l, --language lists used programming languages in given repo
\ No newline at end of file
...@@ -7,14 +7,14 @@ RUN apk update ...@@ -7,14 +7,14 @@ RUN apk update
RUN apk upgrade RUN apk upgrade
# copy entrypoint script to executables # copy entrypoint script to executables
COPY ./build/docker/deploy/docker-entrypoint.sh /usr/local/bin/ COPY ./build/docker/production/docker-entrypoint.sh /usr/local/bin/
COPY ./build/docker/deploy/entrypoint-manual.txt /usr/local/bin/ COPY ./build/docker/production/entrypoint-manual.txt /usr/local/bin/
# copy service scripts to root dir # copy service scripts to root dir
COPY ./src/services /services COPY ./src/services /services
# cleanup # clean up
RUN rm -rf /build RUN rm -rf /services/test
ENTRYPOINT ["docker-entrypoint.sh"] ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["--help"] CMD ["--help"]
...@@ -23,8 +23,16 @@ then ...@@ -23,8 +23,16 @@ then
exit 0 exit 0
fi fi
if [ "$1" = "--language" ] || [ "$1" = "-l" ]
then
/services/language_detection/service_language_detection.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--all" ] || [ "$1" = "-a" ] if [ "$1" = "--all" ] || [ "$1" = "-a" ]
then then
/services/language_detection/service_language_detection.sh
/services/report/service_report.sh "$2" /services/report/service_report.sh "$2"
chmod -R 777 /repo/meta chmod -R 777 /repo/meta
exit 0 exit 0
......
...@@ -6,8 +6,8 @@ Runs various services for software quality detection. ...@@ -6,8 +6,8 @@ Runs various services for software quality detection.
Needs a directory to run services on bound to '/repo' Needs a directory to run services on bound to '/repo'
Available services: Available services:
-h, --help shows this help page
-t, --test show the content of the repo to run services on
and list all available services
and runs all available services with certain debug flags
-a, --all runs all services (reporting is last) -a, --all runs all services (reporting is last)
-h, --help shows this help page
-l, --language lists used programming languages in given repo
-r, --report consolidates reports from previous ran services
generates markdown report
\ No newline at end of file
# Software Quality Report
> **Note:** This report is automatically generated by [ci-services](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services)
---
### Meta Data
**Software Name:** ci-services
**Software Location:** [gitext](https://gitext.gfz-potsdam.de/hifis/software-services/fair/ci-services)
**Last Commit:** 5a47f78a6f329f8c9664657bf933bf6bd03c9ada
**Report Time:** 22/03/2020 01:20:37
---
### Completed Tests
**Programming Language Detection**
> **Description: Detects used programming languages. Output values are bytes of code.**
|name|result|
|---|---|
|Dockerfile|1745|
|Python|4626|
|Ruby|1350|
|Shell|2959|
...@@ -12,6 +12,9 @@ project_name = project_name[0, project_name.length - 5] ...@@ -12,6 +12,9 @@ project_name = project_name[0, project_name.length - 5]
project_location = `echo $CI_PROJECT_URL` project_location = `echo $CI_PROJECT_URL`
project_location = project_location[0, project_location.length - 1] project_location = project_location[0, project_location.length - 1]
if project_location == ''
project_location = 'local'
end
last_commit_ref = `git rev-parse HEAD` last_commit_ref = `git rev-parse HEAD`
last_commit_ref = last_commit_ref[0, 40] last_commit_ref = last_commit_ref[0, 40]
......
...@@ -5,4 +5,3 @@ WORKING_DIR="/meta/language_detection" ...@@ -5,4 +5,3 @@ WORKING_DIR="/meta/language_detection"
cd "$REPO_DIR" || exit 1 cd "$REPO_DIR" || exit 1
ruby /services/language_detection/service_language_detection.rb "$REPO_DIR" "$WORKING_DIR" ruby /services/language_detection/service_language_detection.rb "$REPO_DIR" "$WORKING_DIR"
# bundle exec bin/github-linguist --breakdown
...@@ -72,8 +72,11 @@ with open(working_dir + '/ci-services-report.md', "a+") as md: ...@@ -72,8 +72,11 @@ with open(working_dir + '/ci-services-report.md', "a+") as md:
# write meta data # write meta data
md.write("### Meta Data\n\n") md.write("### Meta Data\n\n")
md.write("**Software Name:** {}\n\n".format(report_dict_ordered['name'])) md.write("**Software Name:** {}\n\n".format(report_dict_ordered['name']))
md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:], if 'https' in report_dict_ordered['location']:
report_dict_ordered['location'])) md.write("**Software Location:** [{}]({})\n\n".format(report_dict_ordered['location'].split('.')[0][8:],
report_dict_ordered['location']))
else:
md.write("**Software Location:** local\n\n")
md.write("**Last Commit:** {}\n\n".format(report_dict_ordered['last commit hash'])) md.write("**Last Commit:** {}\n\n".format(report_dict_ordered['last commit hash']))
md.write("**Report Time:** {}\n\n".format(report_dict_ordered['report time'])) md.write("**Report Time:** {}\n\n".format(report_dict_ordered['report time']))
md.write('---\n\n') md.write('---\n\n')
...@@ -82,7 +85,7 @@ with open(working_dir + '/ci-services-report.md', "a+") as md: ...@@ -82,7 +85,7 @@ with open(working_dir + '/ci-services-report.md', "a+") as md:
md.write("### Completed Tests\n") md.write("### Completed Tests\n")
for subreport in report_dict_ordered: for subreport in report_dict_ordered:
if isinstance(report_dict_ordered[subreport], dict): if isinstance(report_dict_ordered[subreport], dict):
md.write("**{}**\n\n".format(subreport)) md.write("\n**{}**\n\n".format(subreport))
md.write("> **Description: {}**\n\n".format(report_dict_ordered[subreport]['description'])) md.write("> **Description: {}**\n\n".format(report_dict_ordered[subreport]['description']))
# empty table header # empty table header
......
...@@ -11,6 +11,8 @@ working_dir = repo_dir + sys.argv[2] ...@@ -11,6 +11,8 @@ working_dir = repo_dir + sys.argv[2]
project_name = os.popen('basename $(git remote get-url origin)').read()[:-5] project_name = os.popen('basename $(git remote get-url origin)').read()[:-5]
last_commit_ref = os.popen('git rev-parse HEAD').read()[0:40] last_commit_ref = os.popen('git rev-parse HEAD').read()[0:40]
project_location = os.popen('echo $CI_PROJECT_URL').read()[:-1] project_location = os.popen('echo $CI_PROJECT_URL').read()[:-1]
if project_location == '':
project_location = 'local'
report_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") report_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
# generate report as dict # generate report as dict
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
--- ---
### Completed Tests ### Completed Tests
**Programming Language Detection** **Programming Language Detection**
> **Description: Detects used programming languages. Output values are bytes of code.** > **Description: Detects used programming languages. Output values are bytes of code.**
...@@ -26,6 +27,7 @@ ...@@ -26,6 +27,7 @@
|Python|0| |Python|0|
|Ruby|0| |Ruby|0|
|Shell|0| |Shell|0|
**test** **test**
> **Description: used for debug output in test cases** > **Description: used for debug output in test cases**
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment