Commit 42e9acd8 authored by Maximilian Dolling's avatar Maximilian Dolling

first minor release

parent a4b0f57d
.idea
\ No newline at end of file
.idea
meta
\ No newline at end of file
stages:
- build-deploy:dep
- build-deploy
- test
- ci-services
variables:
VERSION: 0.1.0
DOCKER_DST_IMAGE_NAME: ci-services
build-deploy:dep:
stage: build-deploy:dep
image: docker:19.03.1
tags:
- ci-services
variables:
DOCKER_FILE: build/docker/dep/Dockerfile
before_script:
- echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
script:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dep" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dep"
only:
changes:
- "build/dependencies/**/*"
- "build/docker/dep/**/*"
build-deploy:dev:
stage: build-deploy
image: docker:19.03.1
tags:
- ci-services
dependencies:
- build-deploy:dep
variables:
DOCKER_FILE: build/docker/dev/Dockerfile
before_script:
- echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
script:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
except:
refs:
- master
only:
changes:
- "build/**/*"
- "src/**/*"
build-deploy:production:
stage: build-deploy
image: docker:19.03.1
tags:
- ci-services
dependencies:
- build-deploy:dep
variables:
DOCKER_FILE: build/docker/production/Dockerfile
before_script:
- echo "$CI_BUILD_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
script:
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:$VERSION"
- docker build --pull -t "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest" -f $DOCKER_FILE ${DOCKER_BUILD_PATH:-.}
- docker push "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
only:
refs:
- master
changes:
- "build/**/*"
- "src/**/*"
test:dev:
stage: test
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
entrypoint: [""]
tags:
- ci-services
dependencies:
- build-deploy:dev
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh --test
- cp -rf /repo/. "$CI_PROJECT_DIR"
- IS_CHK_SUM=$(sha512sum /repo/meta/ci-services-report.md | awk '{ print $1 }')
- PASS_CHK_SUM=$(sha512sum /repo/test/test_report.md | awk '{ print $1 }')
- >
if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then
echo "Generated report does not equal the test template"
exit 1
fi
except:
- master
only:
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta
when: on_failure
expire_in: 1 day
test:production:
stage: test
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
entrypoint: [""]
tags:
- ci-services
dependencies:
- build-deploy:production
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh --test
- cp -rf /repo/. "$CI_PROJECT_DIR"
- IS_CHK_SUM=$(sha512sum /repo/meta/ci-services-report.md | awk '{ print $1 }')
- PASS_CHK_SUM=$(sha512sum /repo/test/test_report.md | awk '{ print $1 }')
- >
if [ "$IS_CHK_SUM" != "$PASS_CHK_SUM" ]; then
echo "Generated report does not equal the test template"
exit 1
fi
only:
refs:
- master
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta
when: on_failure
expire_in: 1 day
ci-services:dev:
stage: ci-services
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest-dev"
entrypoint: [""]
tags:
- ci-services
dependencies:
- test:dev
before_script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh -a
- cp -rf /repo/meta "$CI_PROJECT_DIR"
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
- git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
- git config user.email "ci-services@gitext.gfz-potsdam.de"
- git add -f meta/ci-services-report.md
- git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
- git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
- exit 0
except:
- master
only:
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta/ci-services-report.md
expire_in: 1 day
ci-services:production:
stage: ci-services
image:
name: "${CI_REGISTRY_IMAGE}/$DOCKER_DST_IMAGE_NAME:latest"
entrypoint: [""]
tags:
- ci-services
dependencies:
- test:production
before_script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS_GITLAB" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh -a
- cp -rf /repo/meta "$CI_PROJECT_DIR"
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY" | tr -d '\r' | ssh-add -
- git config user.name "GitLab CI:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
- git config user.email "ci-services@gitext.gfz-potsdam.de"
- git add -f meta/ci-services-report.md
- git commit -m "[AUTOMATIC] generated meta/ci-services-report.md by gitlab ci"
- git push "git@${CI_SERVER_HOST}:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git" "HEAD:${CI_COMMIT_REF_NAME}"
- exit 0
only:
refs:
- master
changes:
- "build/**/*"
- "src/**/*"
- "test/**/*"
artifacts:
paths:
- meta/ci-services-report.md
expire_in: 1 day
Maximilian Dolling, maximilian.dolling@gfz-potsdam.de
\ No newline at end of file
# CI-Services
This Framework provides various services for the GitLab CI regarding to software quality.
---
Planned Features:
This framework provides various services for the GitLab CI regarding to software quality.
## Description
This software provides a docker image, which provides various services around software quality.
Its goal is to help researchers, developers and [RSEs](https://de-rse.org/en/) develop software with better quality.
Currently there are two ways to run the services.
Either via the [GitLab CI/CD](https://docs.gitlab.com/ee/ci/) (recommended) or a local run.
> **Example:** You can find a full report of all available services [here](meta/ci-services-report.md)
**Current available services:**
* Programming language detection
---
## Requirements
The software you want to run the services on **must** be a [git](https://git-scm.com/) repository!
**via GitLab CI/CD**
* available [GitLab Runner](https://docs.gitlab.com/runner/) with `ci-services` [tag](https://docs.gitlab.com/ee/ci/runners/#using-tags) and the [docker executor](https://docs.gitlab.com/runner/executors/docker.html)
**local usage**
* [docker](https://www.docker.com/) >= 19.03.1
---
## Usage
Running the services via Gitlab CI/CD is the recommended solution.
It keeps the report up to date, every time the project is changed.
You can read more about how it is set up [here](doc/doc_usage_ci.md).
Running it on you local machine is easier to set up.
Therefore it must be triggered manually before every release.
You can read more about how it is set up [here](doc/doc_usage_local.md).
---
## Further planned features
* license check
* license header check
* license generation
* comment check
* language detection
* check for nececary files
* check for necessary files
* generate necessary files
* check if tests exist
* credential check
* DOI request
* curate meta data
* generate report
\ No newline at end of file
---
## Help
TODO =)
\ No newline at end of file
git
bash
openssh-client
\ No newline at end of file
source 'https://rubygems.org'
gem 'github-linguist'
cmake
pkgconfig
icu-dev
zlib-dev
curl-dev
openssl-dev
ruby-dev
g++
make
ruby-bundler
\ No newline at end of file
FROM python:3-alpine
LABEL maintainer="Maximilian Dolling <mdolling@gfz-potsdam.de>"
# update system
RUN apk update
RUN apk upgrade
# copy dependecies
COPY ./build/dependencies /dependencies
# install OS dependencies
RUN apk add --update --no-cache $(cat /dependencies/base/dependencies_OS.txt)
RUN apk add --update --no-cache $(cat /dependencies/language_detection/dependencies_OS.txt)
# install ruby dependencies
RUN bundle install --gemfile=/dependencies/language_detection/Gemfile
# install python dependencies
RUN pip install -r /dependencies/base/requirements.txt
RUN pip install -r /dependencies/language_detection/requirements.txt
# cleanup
RUN rm -rf /dependencies
ENTRYPOINT ["apk add --help"]
\ No newline at end of file
FROM gitext.gfz-potsdam.de:5000/hifis/software-services/fair/ci-services/ci-services:latest-dep
LABEL maintainer="Maximilian Dolling <mdolling@gfz-potsdam.de>"
# update system
RUN apk update
RUN apk upgrade
# copy entrypoint script to executables
COPY ./build/docker/dev/docker-entrypoint.sh /usr/local/bin/
COPY ./build/docker/dev/entrypoint-manual.txt /usr/local/bin/
RUN echo '0.1.0' > /version.txt
# copy service scripts to root dir
COPY ./src/services /services
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["--help"]
#!/usr/bin/env sh
# checks if a repository was bound to the docker image
if [ ! -d /repo ]
then
echo
echo "Repository to run services on not bound! Showing help page..."
cat /usr/local/bin/entrypoint-manual.txt
exit 1
fi
# checks for command line argument to start respective service
if [ "$1" = "--all" ] || [ "$1" = "-a" ]
then
/services/language_detection/service_language_detection.sh
/services/report/service_report.sh "$2"
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--help" ] || [ "$1" = "-h" ]
then
cat /usr/local/bin/entrypoint-manual.txt
exit 0
fi
if [ "$1" = "--language" ] || [ "$1" = "-l" ]
then
/services/language_detection/service_language_detection.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--report" ] || [ "$1" = "-r" ]
then
/services/report/service_report.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--test" ] || [ "$1" = "-t" ]
then
/services/test/service_test.sh
/services/language_detection/service_language_detection.sh
/services/report/service_report.sh --test --debug
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--version" ] || [ "$1" = "-v" ]
then
cat /version.txt
exit 0
fi
# if no suitable argument was found, shows help page
echo
echo "Unknown command line argument! Showing help page..."
cat /usr/local/bin/entrypoint-manual.txt
exit 1
Usage: docker run -v [PATH/TO/REPO]:/repo [SERVICE]
Runs various services for software quality detection.
Needs a directory to run services on bound to '/repo'
Available services:
-a, --all runs all services (reporting is last)
-h, --help shows this help page
-l, --language lists used programming languages in given repo
-r, --report consolidates reports from previous ran services
generates markdown report
-t, --test show the content of the repo to run services on
and list all available services
and runs all available services with certain debug flags
-v, --version returns version of ci-services
\ No newline at end of file
FROM gitext.gfz-potsdam.de:5000/hifis/software-services/fair/ci-services/ci-services:latest-dep
LABEL maintainer="Maximilian Dolling <mdolling@gfz-potsdam.de>"
# update system
RUN apk update
RUN apk upgrade
# copy entrypoint script to executables
COPY ./build/docker/production/docker-entrypoint.sh /usr/local/bin/
COPY ./build/docker/production/entrypoint-manual.txt /usr/local/bin/
RUN echo '0.1.0' > /version.txt
# copy service scripts to root dir
COPY ./src/services /services
# clean up
RUN rm -rf /services/test
ENTRYPOINT ["docker-entrypoint.sh"]
CMD ["--help"]
#!/usr/bin/env sh
# checks if a repository was bound to the docker image
if [ ! -d /repo ]
then
echo
echo "Repository to run services on not bound! Showing help page..."
cat /usr/local/bin/entrypoint-manual.txt
exit 1
fi
# checks for command line argument to start respective service
if [ "$1" = "--all" ] || [ "$1" = "-a" ]
then
/services/language_detection/service_language_detection.sh
/services/report/service_report.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--help" ] || [ "$1" = "-h" ]
then
cat /usr/local/bin/entrypoint-manual.txt
exit 0
fi
if [ "$1" = "--language" ] || [ "$1" = "-l" ]
then
/services/language_detection/service_language_detection.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--report" ] || [ "$1" = "-r" ]
then
/services/report/service_report.sh
chmod -R 777 /repo/meta
exit 0
fi
if [ "$1" = "--version" ] || [ "$1" = "-v" ]
then
cat /version.txt
exit 0
fi
# if no suitable argument was found, shows help page
echo
echo "Unknown command line argument! Showing help page..."
cat /usr/local/bin/entrypoint-manual.txt
exit 1
Usage: docker run -v [PATH/TO/REPO]:/repo [SERVICE]
Runs various services for software quality detection.
Needs a directory to run services on bound to '/repo'
Available services:
-a, --all runs all services (reporting is last)
-h, --help shows this help page
-l, --language lists used programming languages in given repo
-r, --report consolidates reports from previous ran services
generates markdown report
-v, --version returns version of ci-services
\ No newline at end of file
# CI-Services
---
## CI/CD usage
The following manual describes how to use the ci-services with GitLab CI/CD.
It is highly advisable to know how to use git and GitLab when using this method.
> **Note:** It is possible to hook the ci-services to GitHub Actions and other code hosting platforms.
---
## Decide on way to retrieve the report
There are two different ways to get the generated report by the `ci-services`.
**Artifact**
When you opt for this option, the report is available via the GitLab web-interface.
You can simply download.
More information available (here)[https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html]
This option is easier to set up.
**Commit** (recommended)
> **Note:** this option is **required** by certain (future) services.
---
### 1) Project member permissions
To apply certain changes described below, you need at least `Maintainer` [permissions](https://docs.gitlab.com/ee/user/permissions.html).
### 2) Host the project on any GitLab
How to do that, is described [here](https://docs.gitlab.com/ee/gitlab-basics/create-project.html)
### 3) Enable CI/CD
In the GitLab web-interface from the page of your repository navigate to `Settings > General > Visibility, project features, permissions`.
Enable `Pipelines` and set them to `Only Project Members`.
### 4) Add GitLab Runner
To have an available Runner is **required** for the ci-services to function.
This will not be covered here.
Talk to your local admin get one up and running.
**Runner requirements**
* tag: `ci-services`
* executor: docker
> **WARNING:** Be aware that the runtime for the services can go up to several hours, depending on the project size!
Further information are available [here](https://docs.gitlab.com/runner/install/).
---
### 5.1) Retrieve report by Artifact
#### 5.1.1) Set up .gitlab-ci.yml
This yaml block is set up for project without prior GitLab Ci/CD setup.
If you already have `.gitlab-ci.yml` in your project, there should be someone withing your project who can help you adapt.
Copy the content of the following code block to a file named `.gitlab-ci.yml` in the base directory of your project.
Commit and push your changes.
```yaml
stages:
- ci-services
ci-services:
stage: ci-services
image:
name: "gitext.gfz-potsdam.de:5000/hifis/software-services/fair/ci-services/ci-services:latest"
entrypoint: [""]
tags:
- ci-services
script:
- mkdir -p /repo
- cp -r "$CI_PROJECT_DIR/." /repo
- docker-entrypoint.sh -a
- cp -rf /repo/meta "$CI_PROJECT_DIR"
only:
refs:
- master
artifacts:
paths:
- meta/ci-services-report.md
expire_in: 30 days
```
---
### 5.2) Retrieve report by commit
This manual focuses on [UNIX](https://en.wikipedia.org/wiki/Unix) systems like Linux or MacOS.
> **Note:** for Windows the commands can slightly differ, but is fully compatible.
#### 5.2.1) Set up deploy key
First we need to generate a key pair via the terminal.
It will function as [deploy key](https://docs.gitlab.com/ee/ssh/#deploy-keys).
More information on generating a key pair can be found [here](https://docs.gitlab.com/ee/ssh/#generating-a-new-ssh-key-pair).
```console
ssh-keygen -t rsa -b 4096 -C "[PROJECT_NAME]_ci-services" -f [OUTPUTFILE] -q -N ""
```
In the GitLab web-interface from the page of your repository navigate to `Settings > CI/CD > Deploy Keys`.
Paste the content of `[OUTPUTFILE].pub` to the field `key` and give it any unique name (like *PROJECT_NAME*_deploy-key)
Check the 'Write access allowed' checkbox.
Save the changes.
> **Note:** it is recommended to either store `[OUTPUTFILE].pub` securely or delete it.
#### 5.2.2) Set up Variables
We need to tell our CI/CD the identity of the GitLab server we are using.
To do that, we request it from the server via a terminal command.
It returns two lines with the public identity