feat(repos): Add env variable to filter out forks

Refs #92

Author: paveldedik

test/repos/tasks/test_sync_repos.py

@@ -14,15 +14,18 @@ def __init__(self, pid):
 
 
 class FakeGitlabProject(FakeGitProject):
-    def __init__(self, pid, owner, name, url):
+    def __init__(self, pid, owner, name, url, is_fork=False):
         super().__init__(pid)
         self.namespace = {"full_path": self.fake.word() if owner is None else owner}
         self.path = self.fake.word() if name is None else name
         self.web_url = self.fake.url() if url is None else url
 
+        if is_fork:
+            self.forked_from_project = {"id": self.fake.pyint()}
+
 
 class FakeGithubProject(FakeGitProject):
-    def __init__(self, pid, owner, name, url):
+    def __init__(self, pid, owner, name, url, is_fork=False):
         super().__init__(pid)
 
         class FakeOwner:
@@ -33,12 +36,13 @@ def __init__(self, login):
         self.owner = FakeOwner(login)
         self.name = self.fake.word() if name is None else name
         self.svn_url = self.fake.url() if url is None else url
+        self.fork = is_fork
 
 
 def generate_project_list(pid=None, owner=None, name=None, url=None, **kwargs):
     return {
-        "gitlab": [FakeGitlabProject(pid, owner, name, url)],
-        "github": [FakeGithubProject(pid + 1, owner, name, url)],
+        "gitlab": [FakeGitlabProject(pid, owner, name, url, **kwargs)],
+        "github": [FakeGithubProject(pid + 1, owner, name, url, **kwargs)],
     }
 
 
@@ -92,3 +96,30 @@ def test_sync_moved_repo(repository):
     assert github_project.owner.login == repository.owner
     assert github_project.name == repository.name
     assert github_project.svn_url == repository.url
+
+
+def test_sync_skip_forks():
+    project_list = generate_project_list(pid=1, is_fork=True)
+
+    with patch(
+        "gitlab.v4.objects.ProjectManager.list", return_value=project_list["gitlab"]
+    ), patch(
+        "github.AuthenticatedUser.AuthenticatedUser.get_repos",
+        return_value=project_list["github"],
+    ), patch(
+        "zoo.repos.tasks.settings.SYNC_REPOS_SKIP_FORKS", True
+    ):
+        sync_repos()
+
+    gitlab_project = project_list["gitlab"][0]
+    github_project = project_list["github"][0]
+
+    with pytest.raises(Repository.DoesNotExist):
+        Repository.objects.get(
+            remote_id=gitlab_project.id, provider=Provider.GITLAB.value
+        )
+
+    with pytest.raises(Repository.DoesNotExist):
+        Repository.objects.get(
+            remote_id=github_project.id, provider=Provider.GITHUB.value
+        )

zoo/base/settings.py

@@ -38,6 +38,7 @@
     ZOO_SENTRY_URL=(str, None),
     ZOO_SENTRY_ORGANIZATION=(str, None),
     ZOO_SENTRY_API_KEY=(str, None),
+    ZOO_SYNC_REPOS_SKIP_FORKS=(bool, False),
     ZOO_AUDITING_CHECKS=(list, []),
     ZOO_AUDITING_DROP_ISSUES=(int, 7),
     ZOO_SONARQUBE_URL=(str, None),
@@ -251,4 +252,6 @@
 
 GCP_SERVICE_KEY = env("GCP_SERVICE_KEY")
 
+SYNC_REPOS_SKIP_FORKS = env("ZOO_SYNC_REPOS_SKIP_FORKS")
+
 logs.configure_structlog(DEBUG)

zoo/repos/github.py

@@ -24,6 +24,7 @@ def get_repositories():
                 "owner": repo.owner.login,
                 "name": repo.name,
                 "url": repo.svn_url,
+                "is_fork": repo.fork,
             }
     except GithubException:
         log.exception("github.get_repositories.error")

zoo/repos/gitlab.py

@@ -39,6 +39,7 @@ def get_repositories():
                 "owner": project.namespace["full_path"],
                 "name": project.path,
                 "url": project.web_url,
+                "is_fork": hasattr(project, "forked_from_project"),
             }
     except (MissingSchema, GitlabGetError):
         log.exception("gitlab.get_repositories.error")

zoo/repos/tasks.py

@@ -3,6 +3,7 @@
 import tempfile
 
 from celery import shared_task
+from django.conf import settings
 import structlog
 
 from ..analytics.tasks import repo_analyzers
@@ -22,6 +23,8 @@ def sync_repos():
     for project in itertools.chain(
         get_github_repositories(), get_gitlab_repositories()
     ):
+        if settings.SYNC_REPOS_SKIP_FORKS and project["is_fork"]:
+            continue
         try:
             repo = Repository.objects.get(
                 remote_id=project["id"], provider=project["provider"]