Programming Exercises: Experimental support for GitLab CI (ls1intum#6044

)

README.md

@@ -76,7 +76,7 @@ You can find a guide on [how to write documentation](docs/README.md).
 
 ### Server setup
 
-You can set up Artemis in conjunction with either [`GitLab and Jenkins`](https://docs.artemis.cit.tum.de/dev/setup/#jenkins-and-gitlab-setup) or with [`Jira, Bitbucket and Bamboo`](https://docs.artemis.cit.tum.de/dev/setup/#bamboo-bitbucket-and-jira-setup).
+You can set up Artemis in conjunction with either [`GitLab and Jenkins`](https://docs.artemis.cit.tum.de/dev/setup/#jenkins-and-gitlab-setup), [`GitLab and GitLab CI (experimental)`](https://docs.artemis.cit.tum.de/dev/setup/#gitlab-ci-and-gitlab-setup) or with [`Jira, Bitbucket and Bamboo`](https://docs.artemis.cit.tum.de/dev/setup/#bamboo-bitbucket-and-jira-setup).
 Artemis uses these external tools for user management and the configuration of programming exercises.
 
 ### Administration setup

artemis.jh

@@ -94,7 +94,8 @@ entity ProgrammingExercise {        //extends Exercise
     publishBuildPlanUrl Boolean,
     allowOnlineEditor Boolean,
     programmingLanguage ProgrammingLanguage,
-    packageName String
+    packageName String,
+    buildPlanAccessSecret String
 }
 
 enum ProgrammingLanguage {
@@ -109,6 +110,10 @@ enum ProgrammingLanguage {
     OCAML
 }
 
+entity BuildPlan {
+    buildPlan String
+}
+
 entity ModelingExercise {           //extends Exercise
     diagramType DiagramType,
     sampleSolutionModel String,      //LONGTEXT --> lob
@@ -498,7 +503,7 @@ relationship OneToMany {
     ModelAssessmentConflict{resultsInConflict} to ConflictingResult{conflict},
     TextSubmission{blocks} to TextBlock{submission},
     TextCluster{blocks} to TextBlock{cluster},
-    TextExercise{clusters} to TextCluster{exercise}
+    TextExercise{clusters} to TextCluster{exercise},
 }
 
 relationship OneToOne {
@@ -540,6 +545,7 @@ relationship ManyToOne {
     Complaint{student} to User,
     ComplaintResponse{reviewer} to User,
     ModelAssessmentConflict{result} to ExerciseResult,
+    ProgrammingExercise{buildPlan} to BuildPlan,
 }
 
 relationship ManyToMany {

docs/dev/setup.rst

@@ -42,7 +42,7 @@ following dependencies/tools on your machine:
    There are multiple stacks available for the integration with Artemis:
 
    * `GitLab and Jenkins <#jenkins-and-gitlab-setup>`__
-   * GitLab and GitLab CI (under development, not yet production ready)
+   * `GitLab and GitLab CI <#gitlab-ci-and-gitlab-setup>`__ (experimental, not yet production ready)
    * `Bamboo, Bitbucket and Jira <#bamboo-bitbucket-and-jira-setup>`__)
 
 ------------------------------------------------------------------------------------------------------------------------
@@ -520,6 +520,11 @@ instead of the TUM defaults:
 
 ------------------------------------------------------------------------------------------------------------------------
 
+.. include:: setup/gitlabci-gitlab.rst
+
+------------------------------------------------------------------------------------------------------------------------
+
+
 Athene Service
 --------------
 

docs/dev/setup/gitlabci-gitlab.rst

@@ -0,0 +1,137 @@
+GitLab CI and GitLab Setup
+--------------------------
+
+This section describes how to set up a programming exercise environment
+based on GitLab CI and GitLab.
+
+.. note::
+    Depending on your operating system, it might not work with the predefined values (``host.docker.internal``).
+    Therefore, it might be necessary to adapt these with e.g. your local IP address.
+
+**Prerequisites:**
+
+* `Docker <https://docs.docker.com/install>`__
+* `Docker-Compose <https://docs.docker.com/compose/install/>`__
+
+.. contents:: Content of this section
+    :local:
+    :depth: 1
+
+
+GitLab
+^^^^^^
+
+This section describes how to set up a development environment for Artemis with GitLab and GitLab CI.
+The same basic steps as for a `GitLab and Jenkins <#jenkins-and-gitlab-setup>`__ setup apply, but the steps that describe generating tokens for Jenkins can be skipped.
+For a production setup of GitLab, also see the documentation of the GitLab and Jenkins setup.
+
+GitLab
+""""""
+
+1. Depending on your operating system, it is necessary to update the host file of your machine to include the following line:
+
+    .. code:: text
+
+        127.0.0.1       host.docker.internal
+        ::1             host.docker.internal
+
+2. Configure GitLab
+    .. code:: bash
+
+        cp src/main/docker/env.example.gitlab-gitlabci.txt src/main/docker/.env
+
+3. Start GitLab and the GitLab Runner
+    .. code:: bash
+
+        docker-compose -f src/main/docker/gitlab-gitlabci.yml --env-file src/main/docker/.env up --build -d
+
+4. Get your GitLab root password
+    .. code:: bash
+
+        docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password
+
+5. Generate an access token
+    Go to ``http://host.docker.internal/-/profile/personal_access_tokens`` and generate an access token with all scopes.
+    This token is used in the Artemis configuration as ``artemis.version-control.token``.
+
+6. Allow outbound requests to local network
+    For setting up the webhook between Artemis and GitLab, it is necessary to allow requests to the local network.
+    Go to ``http://host.docker.internal/admin/application_settings/network`` and allow the outbound requests.
+    More information about this aspect can be found in the `GitLab setup instructions <#gitlab-access-token>`__ (step 12).
+
+GitLab Runner
+"""""""""""""
+
+1. Register a new runner
+    Login to your GitLab instance and open ``http://host.docker.internal/admin/runners``.
+    Click on ``Register an instance runner`` and copy the registration token.
+
+    Then execute this command with the registration token:
+
+    .. code:: bash
+
+        docker exec -it gitlab-runner gitlab-runner register \
+        --non-interactive \
+        --executor "docker" \
+        --docker-image alpine:latest \
+        --url http://host.docker.internal:80 \
+        --registration-token "PROJECT_REGISTRATION_TOKEN" \
+        --description "docker-runner" \
+        --maintenance-note "Test Runner" \
+        --tag-list "docker,artemis" \
+        --run-untagged="true" \
+        --locked="false" \
+        --access-level="not_protected"
+
+    You should now find the runner in the list of runners (``http://host.docker.internal/admin/runners``)
+
+.. note::
+    Adding a runner in a production setup works the same way.
+    The GitLab administration page also contains alternative ways of setting up GitLab runners.
+    All variants should allow the passing of the configuration options ``tag-list``, ``run-untagged``, ``locked``, and ``access-level`` similarly as in the Docker command above.
+    If forgotten, Artemis might not use this runner to run the tests for exercise submissions.
+
+
+Artemis
+^^^^^^^
+
+.. note::
+    Make sure that the database is empty and contains no data from previous Artemis runs.
+
+1. Generate authentication token
+    The notification plugin has to authenticate to upload the test results.
+    Therefore, a random string has to be generated, e.g., via a password generator.
+    This should be used in place of ``notification-plugin-token`` value in the example config below.
+2. Configure Artemis
+    For local development, copy the following configuration into the ``application-local.yml`` file and adapt it with the values from the previous steps.
+
+    .. code:: yaml
+
+        artemis:
+            user-management:
+                use-external: false
+                internal-admin:
+                    username: artemis_admin
+                    password: gHn7JlggD9YPiarOEJSx19EFp2BDkkq9
+                login:
+                    account-name: TUM
+            version-control:
+                url: http://host.docker.internal:80
+                user: root
+                password: password # change this value
+                token: gitlab-personal-access-token # change this value
+            continuous-integration:
+                build-timeout: 30
+                artemis-authentication-token-value: notification-plugin-token # change this value
+            git:
+                name: Artemis
+                email: [email protected]
+        server:
+            url: http://host.docker.internal:8080
+
+.. note::
+    In GitLab, the password of a user must not be the same as the username and must fulfill specific requirements.
+    Therefore, there is a random password in the example above.
+
+3. Start Artemis
+    Start Artemis with the ``gitlab`` and ``gitlabci`` profile.

src/main/docker/env.example.gitlab-gitlabci.txt

@@ -1,3 +1,3 @@
-GIT_SERVER_NAME=http://git.example.com
+GIT_SERVER_NAME=http://host.docker.internal
 # Make sure to change to https
 SSL_ENABLED=false

src/main/docker/gitlab-gitlabci.yml

@@ -3,6 +3,7 @@ services:
     gitlab:
         build: gitlab
         container_name: gitlab
+        platform: linux/amd64
         volumes:
             - artemis-gitlab-data:/var/opt/gitlab
             - artemis-gitlab-logs:/var/log/gitlab

src/main/java/de/tum/in/www1/artemis/config/RestTemplateConfiguration.java

@@ -31,7 +31,7 @@ public class RestTemplateConfiguration {
     private static final int SHORT_READ_TIMEOUT = 10 * 1000;
 
     @Bean
-    @Profile("gitlab")
+    @Profile("gitlab | gitlabci")
     @Autowired // ok
     public RestTemplate gitlabRestTemplate(GitLabAuthorizationInterceptor gitlabInterceptor) {
         return initializeRestTemplateWithInterceptors(gitlabInterceptor, createRestTemplate());
@@ -79,7 +79,7 @@ public RestTemplate apollonRestTemplate() {
     // it is recommended to keep the timeout settings constant per rest template
 
     @Bean
-    @Profile("gitlab")
+    @Profile("gitlab | gitlabci")
     @Autowired // ok
     public RestTemplate shortTimeoutGitlabRestTemplate(GitLabAuthorizationInterceptor gitlabInterceptor) {
         return initializeRestTemplateWithInterceptors(gitlabInterceptor, createShortTimeoutRestTemplate());

src/main/java/de/tum/in/www1/artemis/config/SecurityConfiguration.java

@@ -170,6 +170,7 @@ protected void configure(HttpSecurity http) throws Exception {
             .antMatchers("/api/account/reset-password/finish").permitAll()
             .antMatchers("/api/lti/launch/*").permitAll()
             .antMatchers("/api/lti13/auth-callback").permitAll()
+            .antMatchers("/api/programming-exercises/*/build-plan").permitAll()
             .antMatchers("/api/**").authenticated()
             .antMatchers("/websocket/tracker").hasAuthority(Role.ADMIN.getAuthority())
             .antMatchers("/websocket/**").permitAll()

src/main/java/de/tum/in/www1/artemis/config/connector/GitLabApiConfiguration.java

@@ -9,7 +9,7 @@
 import org.springframework.context.annotation.Profile;
 
 @Configuration
-@Profile("gitlab")
+@Profile("gitlab | gitlabci")
 public class GitLabApiConfiguration {
 
     @Value("${artemis.version-control.token}")

src/main/java/de/tum/in/www1/artemis/domain/BuildPlan.java

@@ -0,0 +1,37 @@
+package de.tum.in.www1.artemis.domain;
+
+import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Table;
+import javax.persistence.UniqueConstraint;
+import javax.validation.constraints.Size;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+@Entity
+@Table(name = "build_plan", uniqueConstraints = { @UniqueConstraint(columnNames = { "build_plan" }) })
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+public class BuildPlan extends DomainObject {
+
+    @Size(max = 10_000)
+    @Nullable
+    @Column(name = "build_plan", table = "build_plan", length = 10_000)
+    private String buildPlan;
+
+    public BuildPlan() {
+        // explicit constructor needed for Jackson
+    }
+
+    public BuildPlan(String buildPlan) {
+        this.buildPlan = buildPlan;
+    }
+
+    public String getBuildPlan() {
+        return buildPlan;
+    }
+
+    public void setBuildPlan(String buildPlan) {
+        this.buildPlan = buildPlan;
+    }
+}

src/main/java/de/tum/in/www1/artemis/domain/ProgrammingExercise.java

@@ -9,6 +9,7 @@
 
 import javax.annotation.Nullable;
 import javax.persistence.*;
+import javax.validation.constraints.Size;
 
 import org.hibernate.Hibernate;
 import org.slf4j.Logger;
@@ -87,6 +88,15 @@ public class ProgrammingExercise extends Exercise {
     @Column(name = "project_key", table = "programming_exercise_details", nullable = false)
     private String projectKey;
 
+    @Size(max = 36)
+    @Nullable
+    @Column(name = "build_plan_access_secret", table = "programming_exercise_details", length = 36)
+    private String buildPlanAccessSecret;
+
+    @ManyToOne
+    @JoinColumn(name = "build_plan_id", table = "programming_exercise_details")
+    private BuildPlan buildPlan;
+
     @OneToOne(cascade = CascadeType.REMOVE, orphanRemoval = true, fetch = FetchType.LAZY)
     @JoinColumn(unique = true, name = "template_participation_id")
     @JsonIgnoreProperties("programmingExercise")
@@ -811,4 +821,29 @@ public Set<ExerciseHint> getExerciseHints() {
     public void setExerciseHints(Set<ExerciseHint> exerciseHints) {
         this.exerciseHints = exerciseHints;
     }
+
+    public void setBuildPlan(BuildPlan buildPlan) {
+        this.buildPlan = buildPlan;
+    }
+
+    // TODO: Save build plan when exercise is created
+    public BuildPlan getBuildPlan() {
+        return buildPlan;
+    }
+
+    public void setBuildPlanAccessSecret(String buildPlanAccessSecret) {
+        this.buildPlanAccessSecret = buildPlanAccessSecret;
+    }
+
+    public boolean hasBuildPlanAccessSecretSet() {
+        return buildPlanAccessSecret != null && !buildPlanAccessSecret.isEmpty();
+    }
+
+    public String getBuildPlanAccessSecret() {
+        return buildPlanAccessSecret;
+    }
+
+    public void generateAndSetBuildPlanAccessSecret() {
+        buildPlanAccessSecret = UUID.randomUUID().toString();
+    }
 }

src/main/java/de/tum/in/www1/artemis/exception/GitLabCIException.java

@@ -0,0 +1,16 @@
+package de.tum.in.www1.artemis.exception;
+
+public class GitLabCIException extends ContinuousIntegrationException {
+
+    public GitLabCIException(String message) {
+        super(message);
+    }
+
+    public GitLabCIException(Throwable cause) {
+        super(cause);
+    }
+
+    public GitLabCIException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/de/tum/in/www1/artemis/repository/BuildPlanRepository.java

@@ -0,0 +1,12 @@
+package de.tum.in.www1.artemis.repository;
+
+import java.util.Optional;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import de.tum.in.www1.artemis.domain.BuildPlan;
+
+public interface BuildPlanRepository extends JpaRepository<BuildPlan, Long> {
+
+    Optional<BuildPlan> findByBuildPlan(String buildPlan);
+}