Jgrgicevic/sw 593 envs (#342)

* feat: fetch gitlab envs

* fix(services): move gitlab icon to static folder

* refactor: post rebase changes

- change get_or_create usage
- regenerate migrations for envs
- fixed sending NaN on envs fetch

* refactor: rework of gitlab envs

Co-authored-by: Jurica Grgicevic <jurica.grgicevic@kiwi.com>
Co-authored-by: Alex Viscreanu <alexviscreanu@gmail.com>

Author: gjuro87

test/repos/tasks/test_sync_repos.py

@@ -3,12 +3,22 @@
 import pytest
 from faker import Faker
 
-from zoo.repos.models import Provider, Repository
+from zoo.repos.models import Provider, Repository, RepositoryEnvironment
 from zoo.repos.tasks import sync_repos
 
 pytestmark = pytest.mark.django_db
 
 
+class FakeGitlabEnviroment:
+    def __init__(self, id=None, name=None, external_url=None, slug=None):
+        self.fake = Faker()
+        self.id = self.fake.pyint() if id is None else id
+        self.name = self.fake.word() if name is None else name
+        self.slug = self.fake.word() if slug is None else slug
+        self.external_url = self.fake.url() if external_url is None else external_url
+        self.state = "available"
+
+
 class FakeGitProject:
     def __init__(self, pid):
         self.fake = Faker()
@@ -53,38 +63,58 @@ def generate_project_list(pid=None, owner=None, name=None, url=None, **kwargs):
     }
 
 
+def generate_proj_envs_list():
+    return [FakeGitlabEnviroment()]
+
+
 def test_sync_untouched_repo(repository):
     project_list = generate_project_list(
         repository.remote_id, repository.owner, repository.name, repository.url
     )
+    gitlab_envs = generate_proj_envs_list()
     with patch(
         "gitlab.v4.objects.ProjectManager.list", return_value=project_list["gitlab"]
     ), patch(
         "github.AuthenticatedUser.AuthenticatedUser.get_repos",
         return_value=project_list["github"],
+    ), patch(
+        "zoo.repos.tasks.get_project_enviroments",
+        return_value=gitlab_envs,
     ):
         sync_repos()
 
     gitlab_project = project_list["gitlab"][0]
-    assert gitlab_project.id == repository.remote_id
+    repository = Repository.objects.get(
+        remote_id=gitlab_project.id, provider=Provider.GITLAB.value
+    )
     assert gitlab_project.namespace["full_path"] == repository.owner
     assert gitlab_project.path == repository.name
     assert gitlab_project.web_url == repository.url
 
+    repo_env = repository.repository_environments.first()
+    assert gitlab_envs[0].name == repo_env.name
+    assert gitlab_envs[0].external_url == repo_env.external_url
+
     github_project = project_list["github"][0]
-    assert github_project.id == repository.remote_id + 1
+    repository = Repository.objects.get(
+        remote_id=github_project.id, provider=Provider.GITHUB.value
+    )
     assert github_project.owner.login == repository.owner
     assert github_project.name == repository.name
     assert github_project.svn_url == repository.url
 
 
 def test_sync_moved_repo(repository):
     project_list = generate_project_list(repository.remote_id)
+    gitlab_envs = generate_proj_envs_list()
     with patch(
         "gitlab.v4.objects.ProjectManager.list", return_value=project_list["gitlab"]
     ), patch(
         "github.AuthenticatedUser.AuthenticatedUser.get_repos",
         return_value=project_list["github"],
+    ), patch(
+        "zoo.repos.tasks.get_project_enviroments",
+        return_value=gitlab_envs,
     ):
         sync_repos()
 
@@ -96,6 +126,10 @@ def test_sync_moved_repo(repository):
     assert gitlab_project.path == repository.name
     assert gitlab_project.web_url == repository.url
 
+    repo_env = repository.repository_environments.first()
+    assert gitlab_envs[0].name == repo_env.name
+    assert gitlab_envs[0].external_url == repo_env.external_url
+
     github_project = project_list["github"][0]
     repository = Repository.objects.get(
         remote_id=github_project.id, provider=Provider.GITHUB.value

test/services/test_forms.py

@@ -116,11 +116,13 @@ def test_service_form__complete__incorrect_status(repository):
     "environments-0-service_urls_0": fake.url(),
     "environments-0-service_urls_1": fake.url(),
     "environments-0-DELETE": False,
+    "environments-0-type": "zoo",
     "environments-1-name": fake.word(),
     "environments-1-dashboard_url": fake.url(),
     "environments-1-service_urls_0": fake.url(),
     "environments-1-service_urls_1": fake.url(),
     "environments-1-DELETE": False,
+    "environments-1-type": "zoo",
 }
 
 

zoo/base/assets/js/components/RepoInput.vue

@@ -36,6 +36,8 @@
 import { match } from "ramda"
 import RepoInputSuggestions from "./RepoInputSuggestions"
 import $ from 'jquery/src/jquery'
+import { gitlabEnvs } from "../../../../services/assets/js/gitlab_envs.js"
+
 
 function createPopup () {
   $('i.magic.icon').popup({
@@ -51,6 +53,11 @@ function popupAction (action) {
   $('i.magic.icon').popup(action)
 }
 
+function loadGitlabEnvs(repoId) {
+  gitlabEnvsInfo["repoId"] = repoId
+  gitlabEnvs.load(gitlabEnvsInfo)
+}
+
 export default {
   template: "#repo-input-text-field-markup",
   data () {
@@ -153,6 +160,7 @@ export default {
       this.$refs.suggestions.selectSuggestion();
       this.isOnEditMode = false
       this.$store.commit("refreshEnteredText")
+      loadGitlabEnvs(parseInt(this.inputValue))
     },
   },
   components: {

zoo/factories.py

@@ -18,7 +18,7 @@
 from zoo.auditing.check_discovery import Kind
 from zoo.auditing.models import Issue
 from zoo.datacenters.models import InfraNode
-from zoo.repos.models import Repository
+from zoo.repos.models import Repository, RepositoryEnvironment
 from zoo.services.models import Environment, Impact, Link, Service, Status, Tier
 
 
@@ -33,6 +33,15 @@ class Meta:
     url = LazyAttribute(lambda o: f"https://gitlab.com/{o.owner}/{o.name}")
 
 
+class RepositoryEnvironmentFactory(DjangoModelFactory):
+    class Meta:
+        model = RepositoryEnvironment
+
+    name = Faker("domain_word")
+    repository = SubFactory(RepositoryFactory)
+    external_url = LazyAttribute(lambda o: f"https://gitlab.com/{o.name}/{o.name}")
+
+
 class IssueFactory(DjangoModelFactory):
     class Meta:
         model = Issue

zoo/repos/admin.py

@@ -6,3 +6,8 @@
 @admin.register(models.Repository)
 class RepoAdmin(admin.ModelAdmin):
     search_fields = ("remote_id", "owner", "name", "provider")
+
+
+@admin.register(models.RepositoryEnvironment)
+class RepoEnvAdmin(admin.ModelAdmin):
+    search_fields = ("name", "external_url")

zoo/repos/forms.py

@@ -1,5 +1,6 @@
 from django.forms import widgets
 
+from ..services.constants import EnviromentType
 from .models import Repository
 
 
@@ -19,4 +20,5 @@ def get_context(self, name, value, attrs):
         )
 
         context["widget"]["value"] = value
+        context["env_type_gitlab"] = EnviromentType.GITLAB.value
         return context

zoo/repos/gitlab.py

@@ -13,6 +13,14 @@
 log = structlog.get_logger()
 
 
+def get_project_enviroments(remote_id):
+    try:
+        project = gitlab.projects.get(remote_id)
+        return project.environments.list(as_list=False)
+    except (GitlabGetError, GitlabListError):
+        return []
+
+
 def get_project(remote_id):
     try:
         project = gitlab.projects.get(remote_id)

zoo/repos/migrations/0008_repositoryenvironment.py

@@ -0,0 +1,43 @@
+# Generated by Django 2.2.19 on 2021-04-30 14:24
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("repos", "0007_endpoint"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RepositoryEnvironment",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=200)),
+                ("external_url", models.CharField(max_length=300, null=True)),
+                (
+                    "repository",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="repository_environments",
+                        related_query_name="repository_environment",
+                        to="repos.Repository",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["name"],
+                "unique_together": {("repository", "name")},
+            },
+        ),
+    ]

zoo/repos/models.py

@@ -86,3 +86,19 @@ class Meta:
     )
     summary = models.CharField(max_length=500, null=True, blank=True)
     operation = models.CharField(max_length=200, null=True, blank=True)
+
+
+class RepositoryEnvironment(models.Model):
+    class Meta:
+        unique_together = ("repository", "name")
+        ordering = ["name"]
+
+    repository = models.ForeignKey(
+        Repository,
+        on_delete=models.CASCADE,
+        related_name="repository_environments",
+        related_query_name="repository_environment",
+    )
+
+    name = models.CharField(max_length=200)
+    external_url = models.CharField(max_length=300, null=True)

zoo/repos/tasks.py

@@ -12,11 +12,13 @@
 from ..auditing import runner
 from ..auditing.check_discovery import CHECKS as AUDITING_CHECKS
 from ..repos.models import Endpoint
+from ..services.constants import EnviromentType
 from ..services.models import Environment, Service
 from .exceptions import MissingFilesError, RepositoryNotFoundError
 from .github import get_repositories as get_github_repositories
+from .gitlab import get_project_enviroments
 from .gitlab import get_repositories as get_gitlab_repositories
-from .models import Repository
+from .models import Repository, RepositoryEnvironment
 from .utils import download_repository, get_scm_module, openapi_definition
 from .zoo_yml import parse, validate
 
@@ -39,14 +41,17 @@
 
 @shared_task
 def sync_repos():
+
+    i = 0
     for project in itertools.chain(
         get_github_repositories(), get_gitlab_repositories()
     ):
         if settings.SYNC_REPOS_SKIP_FORKS and project["is_fork"]:
             continue
         if settings.SYNC_REPOS_SKIP_PERSONAL and project["is_personal"]:
             continue
-
+        i += 1
+        log.info("sync_repos.fetch", repo_number=i, project=project)
         try:
             repo = Repository.objects.get(
                 remote_id=project["id"], provider=project["provider"]
@@ -68,6 +73,12 @@ def sync_repos():
         repo.full_clean()
         repo.save()
 
+        if project["provider"] == "gitlab":
+            log.info("sync_repos.calling.sync_enviroments_from_gitlab")
+            sync_enviroments_from_gitlab(repo)
+
+    log.info("sync_repos.total", repo_number=i)
+
 
 @shared_task
 def schedule_pulls():
@@ -208,3 +219,46 @@ def get_zoo_file_content(proj: Dict) -> str:
     return provider.get_file_content(
         proj["id"], settings.ZOO_YAML_FILE, settings.ZOO_YAML_DEFAULT_REF
     )
+
+
+def sync_enviroments_from_gitlab(repo: Repository):
+    gl_envs = get_project_enviroments(repo.remote_id)
+
+    if not gl_envs:
+        log.info("sync_enviroments_from_gitlab.no_envs")
+        return
+
+    envs = []
+    i = 0
+    log.info("sync_enviroments_from_gitlab.start")
+    for gl_env in gl_envs:
+        if gl_env.state != "available":
+            continue
+        env, _ = RepositoryEnvironment.objects.get_or_create(
+            repository_id=repo.id, name=gl_env.name
+        )
+        env.external_url = gl_env.external_url
+        env.save()
+        envs.append(env)
+        i += 1
+
+    RepositoryEnvironment.objects.filter(repository_id=repo.id).exclude(
+        id__in=[env.id for env in envs]
+    ).delete()
+
+    # update gitlab envs on every service
+    services = Service.objects.filter(repository_id=repo.id)
+    for service in services:
+        Environment.objects.filter(
+            service_id=service.id, type=EnviromentType.GITLAB.value
+        ).exclude(name__in=[env.name for env in envs]).delete()
+        for env in envs:
+            service_env, _ = Environment.objects.get_or_create(
+                service_id=service.id,
+                name=env.name,
+                type=EnviromentType.GITLAB.value,
+            )
+            service_env.dashboard_url = env.external_url
+            service_env.save()
+
+    log.info("sync_enviroments_from_gitlab.total.synced", envs_synced=i)