diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5ac3e00 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +# EditorConfig is awesome: https://EditorConfig.org +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.py] +indent_style = space +indent_size = 4 +charset = utf-8 + +[Makefile] +indent_style = tab + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index bbd194c..5d165c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .vscode -*.json -.repos .idea __pycache__ -ENV -test.py \ No newline at end of file +.venv* +build +dist +*.egg-info diff --git a/LICENSE b/LICENSE index fb332c1..be250f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020-2021 hsowan +Copyright (c) 2020 K8sCat and The Gigrator Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1343eab --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +version = 1.0.2 + +install: + python setup.py install + +.PHONY: install-requirements +install-requirements: + pip install -r requirements.txt + +build-wheel: clean + sed -i 's/{version}/$(version)/g' setup.py + python setup.py sdist bdist_wheel + +username = __token__ +password = +release-pypi: install-requirements clean build-wheel + python -m twine upload -u $(username) -p $(password) dist/* + +clean: + rm -rf build dist gigrator.egg-info + +install-wheel: + pip install --force-reinstall dist/gigrator-$(version)-py3-none-any.whl diff --git a/README.md b/README.md index 64117ab..6921f60 100644 --- a/README.md +++ b/README.md @@ -1,123 +1,87 @@ # Gigrator -[![](https://img.shields.io/badge/GitHub-success)](https://github.com/k8scat/gigrator) -[![](https://img.shields.io/badge/Gitee-red)](https://gitee.com/k8scat/gigrator) +Gigrator 是一个 Git 代码仓批量迁移工具,支持众多流行的代码托管平台,包括 GitHub、码云(Gitee)、GitLab、Gitea、Coding、Gogs、腾讯工蜂, +同时可以基于本项目进行拓展其他代码托管平台。 -Git 代码仓批量迁移 +如果你在国外 GitHub 上托管了数十上百个代码仓, +现在想要快速地迁移到国内的码云(Gitee)或自建的 Git 代码托管平台(比如 GitLab、Gitea)上的话, +可以使用 [Gigrator](https://github.com/k8scat/gigrator.git) 帮助你快速地迁移这些代码仓。 -![gigrator.png](images/gigrator.png) +## 支持的平台 -## Start +- [x] [Gitee](https://gitee.com/) +- [x] [GitLab](https://gitlab.com/) +- [x] [GitHub](https://github.com/) +- [x] [Gitea](https://gitea.io/zh-cn/) +- [x] [Coding](https://coding.net/) +- [x] [Gogs](https://gogs.io/) +- [x] [腾讯工蜂](https://code.tencent.com/) +- [ ] [Bitbucket](https://bitbucket.org/) +- [ ] [云效 Codeup](https://codeup.aliyun.com/) -```shell script -git clone https://github.com/k8scat/gigrator.git -cd gigrator -pip3 install -r requirements.txt +说明: -# 迁移前需在配置文件(settings.py)中配置 SOURCE_GIT 和 DEST_GIT -# 配置参考: settings_example.py -python3 gigrator.py -``` +- 暂不支持迁移至 `Coding`,可从 Coding 迁移至其他 `Git` 服务器 +- 由于 `Coding` 的升级,其基础 `API` 不再是 `https://coding.net`,而改为: `https://{username}.coding.net` +- 迁移前请确认已在Git服务器上添加 `SSH Key` +- 只能迁移指定用户下的仓库,即 `{username}/{repo_name}`,不包括参与的或者组织的仓库 +- 迁移包括commits、branches和tags,不包括issues、pr和wiki -## Develop +## 安装 -```python -# Base class -class Git: - pass +### pip 安装 -# Other GitServer class should inherit Git -class OtherGit(Git): - pass +```bash +pip install gigrator ``` -## Support - -* [x] [Gitee](https://gitee.com/) -* [x] [GitLab](https://gitlab.com/) -* [x] [GitHub](https://github.com/) -* [x] [Gitea](https://gitea.io/zh-cn/) -* [x] [Coding](https://coding.net/) -* [x] [Gogs](https://gogs.io/) -* [x] [腾讯工蜂](https://code.tencent.com/) -* [ ] [Bitbucket](https://bitbucket.org/) - -Note: - -* 不支持迁移至 `Coding`, 可从 Coding 迁移至其他 `Git` 服务器 -* 由于 `Coding` 的升级, 其基础 `API` 不再是 `https://coding.net`, 而改为: `https://{username}.coding.net` -* 迁移前请确认已在Git服务器上添加 `SSH Key` -* 只能迁移指定用户下的仓库, 即 `{username}/{repo_name}`, 不包括参与的或者组织的仓库 -* 迁移包括commits、branches和tags, 不包括issues、pr和wiki - -## Environment - -* Git -* Python - -开发环境: `git version 2.20.1 (Apple Git-117)` + `Python 3.7.2` - -## Dependencies - -* [Requests](https://2.python-requests.org/en/master/) - -## References - -### GitLab +### 源码安装 -* [GitLab API Docs](https://docs.gitlab.com/ee/api/) -* [GitLab Create Repo](https://docs.gitlab.com/ee/api/projects.html#create-project) -* [Project visibility level](https://docs.gitlab.com/ee/api/projects.html#project-visibility-level) - -## [GitLab GraphQL API](https://docs.gitlab.com/ee/api/graphql/) - -Can not create a project! - -It will co-exist with the current v4 REST API. If we have a v5 API, this should be a compatibility layer on top of GraphQL. - -* [Introduction to GraphQL](https://developer.github.com/v4/guides/intro-to-graphql/) -* [GraphQL API Resources](https://docs.gitlab.com/ee/api/graphql/reference/index.html) - -### [GitHub REST API v3](https://developer.github.com/v3/) - -* [GitHub Create Repo](https://developer.github.com/v3/repos/#create) -* [GitHub Personal Access Token](https://github.com/settings/tokens) - -## [GitHub GraphQL API v4](https://developer.github.com/v4/) +```shell script +git clone https://github.com/k8scat/gigrator.git +cd gigrator +make +``` -* [GraphQL resource limitations](https://developer.github.com/v4/guides/resource-limitations/) -* [Forming Calls with GraphQL](https://developer.github.com/v4/guides/forming-calls/) +## 使用 +### 环境要求 -### Gitee +- Git +- Python3 -* [Gitee OpenAPI](https://gitee.com/api/v5/swagger#/getV5ReposOwnerRepoStargazers?ex=no) -* [Gitee Personal Access Token](https://gitee.com/profile/personal_access_tokens) +### 配置文件 -### Gitea +参考 [config.yml](./config.yml) -* [Gitea API](https://gitea.com/api/v1/swagger) -* [Get a repo](https://gitea.com/api/v1/swagger#/repository/repoGet) -* [Create a repo](https://gitea.com/api/v1/swagger#/repository/createCurrentUserRepo) -* [List the repos that the authenticated user owns or has access to](https://gitea.com/api/v1/swagger#/user/userCurrentListRepos) +### 运行 -### Gogs +```bash +gigrator -c config.yml +``` -* [gogs/docs-api](https://github.com/gogs/docs-api) -* [Demo site](https://try.gogs.io/) +## 扩展更多平台 -### Coding +基于 `Git` 类实现其他平台的代码仓迁移 -* [Open API](https://open.coding.net/open-api/?_ga=2.122224323.99121124.1563808661-1235584671.1544277191) +```python +class Git: + def list_repos(self) -> list: + raise NotImplementedError -### GF (腾讯工蜂) + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + raise NotImplementedError -* [Open API](https://code.tencent.com/help/api/prepare) + def is_repo_existed(self, repo_name: str) -> bool: + raise NotImplementedError +``` -### GraphQL Client +## 贡献 -* [sgqlc](https://github.com/profusion/sgqlc) + + + -## License +## 开源协议 LICENSE [MIT](https://github.com/k8scat/gigrator/blob/master/LICENSE) diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..5ecfccd --- /dev/null +++ b/config.yml @@ -0,0 +1,57 @@ +migrate: + from: github + to: gitee + clone_dir: "" + # 默认迁移所有代码仓,可以设置 repos 迁移特定的代码仓 + # repos: + # - repo_a + # - repo_b + +gitee: + provider: gitee + api: https://gitee.com/api/v5 + ssh_prefix: git@gitee.com + https_prefix: https://gitee.com + username: "" + token: "" + +github: + provider: github + api: https://api.github.com/graphql + ssh_prefix: git@github.com + https_prefix: https://github.com + username: "" + token: "" + +gitlab: + provider": gitlab + api: https://gitlab.com/api/v4 + ssh_prefix: git@gitlab.com + https_prefix: https://gitlab.com + username: "" + token: "" + +coding: + provider: coding + api: https://{your-team}.coding.net + ssh_prefix: git@e.coding.net + https_prefix: https://e.coding.net + username: "" + token: "" + +# 腾讯工蜂 https://code.tencent.com/ +gongfeng: + provider: gongfeng + api: https://code.tencent.com/api/v3 + ssh_prefix: git@git.code.tencent.com + https_prefix: https://git.code.tencent.com + username: "" + token: "" + +gitea: + provider: gitea + api: https://gitea.com/api/v1 + ssh_prefix: git@gitea.com + https_prefix: https://gitea.com + username: "" + token: "" diff --git a/dev.md b/dev.md new file mode 100644 index 0000000..4266c5d --- /dev/null +++ b/dev.md @@ -0,0 +1,47 @@ +# 开发文档 + +## GitLab + +- [GitLab API Docs](https://docs.gitlab.com/ee/api/) +- [GitLab Create Repo](https://docs.gitlab.com/ee/api/projects.html#create-project) +- [Project visibility level](https://docs.gitlab.com/ee/api/projects.html#project-visibility-level) +- [GitLab GraphQL API](https://docs.gitlab.com/ee/api/graphql/) +- [Introduction to GraphQL](https://developer.github.com/v4/guides/intro-to-graphql/) +- [GraphQL API Resources](https://docs.gitlab.com/ee/api/graphql/reference/index.html) + +## [GitHub REST API v3](https://developer.github.com/v3/) + +- [GitHub Create Repo](https://developer.github.com/v3/repos/#create) +- [GitHub Personal Access Token](https://github.com/settings/tokens) +- [GitHub GraphQL API v4](https://developer.github.com/v4/) +- [GraphQL resource limitations](https://developer.github.com/v4/guides/resource-limitations/) +- [Forming Calls with GraphQL](https://developer.github.com/v4/guides/forming-calls/) + +## Gitee + +- [Gitee OpenAPI](https://gitee.com/api/v5/swagger#/getV5ReposOwnerRepoStargazers?ex=no) +- [Gitee Personal Access Token](https://gitee.com/profile/personal_access_tokens) + +## Gitea + +- [Gitea API](https://gitea.com/api/v1/swagger) +- [Get a repo](https://gitea.com/api/v1/swagger#/repository/repoGet) +- [Create a repo](https://gitea.com/api/v1/swagger#/repository/createCurrentUserRepo) +- [List the repos that the authenticated user owns or has access to](https://gitea.com/api/v1/swagger#/user/userCurrentListRepos) + +## Gogs + +- [gogs/docs-api](https://github.com/gogs/docs-api) +- [Demo site](https://try.gogs.io/) + +## Coding + +- [Open API](https://help.coding.net/openapi) + +## GF (腾讯工蜂) + +- [Open API](https://code.tencent.com/help/api/prepare) + +## GraphQL Client + +- [sgqlc](https://github.com/profusion/sgqlc) diff --git a/gigrator.py b/gigrator.py deleted file mode 100644 index 9658bc7..0000000 --- a/gigrator.py +++ /dev/null @@ -1,574 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -@author: hsowan -@date: 2020/2/10 - -""" -import json -import logging -import os -import re -import uuid -from urllib.parse import quote, urlencode -import requests -import settings - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -class Git: - type = '' - username = '' - token = '' - url = '' - api = '' - ssh_prefix = '' - headers = {} - - def __init__(self, config: dict): - self.type = config['type'].lower() - if self.type == 'gogs': - self.type = 'gitea' - self.username = config['username'] - self.token = config['token'] - self.url = config['url'] - - if not self.type: - raise ValueError('type 必须配置') - - if self.type not in settings.SUPPORT_GITS: - raise ValueError('暂不支持此类型的Git服务器: ' + self.type) - - if self.type in ['gitlab', 'gitea'] and not self.url: - raise ValueError('gitlab/gitea/gogs 需要配置url') - - if not self.username: - raise ValueError('username 必须配置') - - if not self.token: - raise ValueError('token 必须配置') - - if self.type in ['gitlab', 'gitea']: - if not re.match(r'^http(s)?://.+$', self.url): - raise ValueError('url 配置有误') - if self.url.endswith('/'): - self.url = self.url[:-1] - - def is_existed(self, repo_name: str) -> bool: - """ - Check repo existed or not - - :param repo_name: - :return: - """ - raise NotImplementedError - - def clone_repo(self, repo_name: str) -> str: - """ - Clone repo - - :param repo_name: - :return: the local dir of saved repo - """ - - clone_space = os.path.join(settings.TEMP_DIR, str(uuid.uuid1())) - os.mkdir(clone_space) - - # 检查本地是否存在repo, 存在则删除 - ssh_address = self.ssh_prefix + self.username + '/' + repo_name + '.git' - clone_cmd = 'cd ' + clone_space + ' && git clone --bare ' + ssh_address - - return os.path.join(clone_space, repo_name + '.git') if os.system(clone_cmd) == 0 else None - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - """ - Create repo - - :param name: - :param desc: - :param is_private: - :return: create successfully or not - """ - raise NotImplementedError - - def push_repo(self, repo_name: str, repo_dir: str) -> bool: - """ - Push repo - - :param repo_name: - :param repo_dir: - :return: bool - """ - ssh_address = self.ssh_prefix + self.username + '/' + repo_name + '.git' - push_cmd = 'cd ' + repo_dir + ' && git push --mirror ' + ssh_address - return os.system(push_cmd) == 0 - - def list_repos(self) -> list: - """ - List all repos owned - - :return: a list of repos - """ - raise NotImplementedError - - -class Gitlab(Git): - - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'Private-Token': self.token - } - self.ssh_prefix = 'git@' + self.url.split('://')[1] + ':' - self.api = self.url + settings.GITLAB_API_VERSION - - def is_existed(self, repo_name: str) -> bool: - # Get single project - # GET /projects/:id - path = quote(f'{self.username}/{repo_name}', safe='') - url = f'{self.api}/projects/{path}' - r = requests.get(url, headers=self.headers) - return r.status_code == 200 - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - data = { - 'name': name, - 'description': desc, - 'visibility': 'private' if is_private else 'public' - } - url = f'{self.api}/projects' - r = requests.post(url, data=json.dumps(data), headers=self.headers) - return r.status_code == 201 - - def list_repos(self) -> list: - # GitLab - # List user projects: GET /users/:user_id/projects (需要分页: ?page=1) - # 不存在401的问题: 会返回公开的仓库 - url = f'{self.api}/users/{self.username}/projects?page=' - # 当没有授权时, 可能只会返回公开项目(至少gitlab会) - all_repos = [] - page = 1 - while True: - r = requests.get(url + str(page), headers=self.headers) - if r.status_code == 200: - repos = r.json() - if len(repos) == 0: - break - for repo in repos: - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=repo['visibility'] != 'public')) - page += 1 - else: - raise RuntimeError(r.content) - return all_repos - - -class Github(Git): - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'Authorization': 'token ' + self.token - } - self.ssh_prefix = settings.GITHUB_SSH_PREFIX - self.api = settings.GITHUB_API - - def is_existed(self, repo_name: str) -> bool: - query = ''' - query ($repo_owner: String!, $repo_name: String!) { - repository(owner: $repo_owner, name: $repo_name) { - id - } - } - ''' - variables = { - 'repo_owner': self.username, - 'repo_name': repo_name - } - post_data = json.dumps({ - 'query': query, - 'variables': variables - }) - r = requests.post(self.api, data=post_data, headers=self.headers) - if r.status_code == 200: - data = r.json() - try: - return data['data']['repository'].get('id', None) is not None - except KeyError: - return False - else: - raise RuntimeError(r.content) - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - mutation = ''' - mutation ($name: String!, $desc: String!, $isPrivate: RepositoryVisibility!) { - createRepository(input: {name: $name, description: $desc, visibility: $isPrivate}) { - clientMutationId - repository { - id - } - } - } - ''' - variables = { - 'name': name, - 'desc': desc, - 'isPrivate': 'PRIVATE' if is_private else 'PUBLIC' - } - post_data = json.dumps({ - 'query': mutation, - 'variables': variables - }) - r = requests.post(self.api, data=post_data, headers=self.headers) - if r.status_code == 200: - data = r.json() - return 'errors' not in data.keys() - else: - raise RuntimeError(r.content) - - def list_repos(self) -> list: - query = ''' - query ($first: Int!, $after: String) { - viewer { - repositories(first: $first, after: $after, ownerAffiliations: [OWNER]) { - edges { - node { - name - isPrivate - description - } - cursor - } - pageInfo { - hasNextPage - } - } - } - } - ''' - variables = { - 'first': 100 - } - post_data = json.dumps({ - 'query': query, - 'variables': variables - }) - r = requests.post(self.api, data=post_data, headers=self.headers) - if r.status_code == 200: - data = r.json() - all_repos = [] - try: - def parse_data(): - repos = data['data']['viewer']['repositories']['edges'] - for repo in repos: - repo = repo['node'] - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=repo['isPrivate'])) - parse_data() - has_next_page = data['data']['viewer']['repositories']['pageInfo']['hasNextPage'] - while has_next_page: - variables['after'] = data['data']['viewer']['repositories']['edges'][-1]['cursor'] - post_data = json.dumps({ - 'query': query, - 'variables': variables - }) - r = requests.post(self.api, data=post_data, headers=self.headers) - if r.status_code == 200: - data = r.json() - parse_data() - has_next_page = data['data']['viewer']['repositories']['pageInfo']['hasNextPage'] - else: - raise RuntimeError(r.content) - except KeyError: - logger.error(data) - finally: - return all_repos - else: - raise RuntimeError(r.content) - - -class Gitee(Git): - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'Content-Type': 'application/json;charset=UTF-8' - } - self.ssh_prefix = settings.GITEE_SSH_PREFIX - self.api = settings.GITEE_API - - def is_existed(self, repo_name: str) -> bool: - # 获取用户的某个仓库: GET /repos/{owner}/{repo} - # https://gitee.com/api/v5/swagger#/getV5ReposOwnerRepo - url = f'{self.api}/repos/{self.username}/{repo_name}?access_token={self.token}' - r = requests.get(url, headers=self.headers) - return r.status_code == 200 - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - data = { - 'access_token': self.token, - 'name': name, - 'description': desc, - 'private': is_private - } - url = f'{self.api}/user/repos' - r = requests.post(url, json=data, headers=self.headers) - return r.status_code == 201 - - def list_repos(self) -> list: - # Gitee - # 列出授权用户的所有仓库: GET /user/repos - # https://gitee.com/api/v5/swagger#/getV5UserRepos - list_repos_url = self.api + '/user/repos?access_token=' + self.token \ - + '&type=personal&sort=full_name&per_page=100&page=' - page = 1 - all_repos = [] - while True: - r = requests.get(list_repos_url + str(page), headers=self.headers) - if r.status_code == 200: - repos = r.json() - if len(repos) == 0: - break - for repo in repos: - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=repo['private'])) - page += 1 - elif r.status_code == 401: - raise ValueError('token 无效') - else: - raise RuntimeError(r.content.decode('utf-8')) - return all_repos - - -class Gitea(Git): - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'Content-Type': 'application/json', - 'Authorization': f'token {self.token}' - } - self.ssh_prefix = f'git@{self.url.split("://")[1]}:' - self.api = self.url + settings.GITEA_API_VERSION - - def is_existed(self, repo_name: str) -> bool: - # GET - # ​/repos​/{owner}​/{repo} - # Get a repository - url = f'{self.api}/repos/{self.username}/{repo_name}' - r = requests.get(url, headers=self.headers) - return r.status_code == 200 - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - data = { - 'auto_init': False, - 'description': desc, - 'name': name, - 'private': is_private - } - url = f'{self.api}/user/repos' - r = requests.post(url, headers=self.headers, data=json.dumps(data)) - return r.status_code == 201 - - def list_repos(self) -> list: - # GET - # ​/user​/repos - # List the repos that the authenticated user owns or has access to - # 没有做分页: https://github.com/go-gitea/gitea/issues/7515 - list_repos_url = self.api + '/user/repos' - all_repos = [] - r = requests.get(list_repos_url, headers=self.headers) - if r.status_code == 200: - repos = r.json() - for repo in repos: - if repo['owner']['username'] == self.username: - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=repo['private'])) - return all_repos - - -class Coding(Git): - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'Authorization': 'token ' + self.token - } - self.ssh_prefix = settings.CODING_SSH_PREFIX - self.api = f'https://{self.username}.coding.net' - - def is_existed(self, repo_name: str) -> bool: - # GET /api/user/{username}/project/{project_name} - url = f'{self.api}/api/user/{self.username}/project/{repo_name}' - r = requests.get(url, headers=self.headers) - if r.status_code == 200: - data = r.json() - return data['code'] == 0 and data['data']['name'] == repo_name - else: - return False - - def create_repo(self, name: str, desc: str, is_private: bool) -> bool: - raise PermissionError('Coding 不支持通过API创建仓库') - - def list_repos(self) -> list: - # 当前用户的项目列表 - # https://open.coding.net/api-reference/%E9%A1%B9%E7%9B%AE.html#%E5%BD%93%E5%89%8D%E7%94%A8%E6%88%B7%E7%9A%84%E9%A1%B9%E7%9B%AE%E5%88%97%E8%A1%A8 - # GET /api/user/projects?type=all&page={page}&pageSize={pageSize} - # Response包含totalPage - url = f'{self.api}/api/user/projects?type=all&pageSize=10&page=' - page = 1 - all_repos = [] - r = requests.get(url + str(page), headers=self.headers) - if r.status_code == 200: - data = r.json() - if data['code'] == 0: - total_page = data['data']['totalPage'] - repos = data['data']['list'] - for repo in repos: - if str.lower(repo['owner_user_name']) == str.lower(self.username): - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=not repo['is_public'])) - else: - raise RuntimeError(data) - - while page < total_page: - page += 1 - r = requests.get(url + str(page), headers=self.headers) - if r.status_code == 200: - if data['code'] == 0: - data = r.json() - repos = data['data']['list'] - for repo in repos: - if str.lower(repo['owner_user_name']) == str.lower(self.username): - all_repos.append(dict(name=repo['name'], - desc=repo['description'], - is_private=not repo['is_public'])) - else: - raise RuntimeError(data) - - return all_repos - - -# 腾讯工蜂 -class GF(Git): - def __init__(self, config: dict): - super().__init__(config) - self.headers = { - 'PRIVATE-TOKEN': self.token - } - self.ssh_prefix = settings.GF_SSH_PREFIX - self.api = settings.GF_API - - def is_existed(self, repo_name: str) -> bool: - # repo_name如果包含命名空间需要URL编码 - repo_name_encoded = urlencode(repo_name) - url = f'{self.api}/projects/{repo_name_encoded}/repository/tree' - r = requests.get(url, headers=self.headers) - if r.status_code == 200: - return True - else: - return False - - # repo_name = name_with_namespace - def create_repo(self, repo_name: str, desc: str, is_private: bool) -> bool: - url = self.api + "/projects" - repo_full_name_list = repo_name.split("/") - if len(repo_full_name_list) == 1: - repo_name_namespace = "" - repo_name_short = repo_name - else: - repo_name_namespace = repo_full_name_list[0] - repo_name_short = repo_full_name_list[1] - if is_private: - visibility_level = 0 - else: - visibility_level = 10 - data = { - "name": repo_name_short, - "namespace_id": repo_name_namespace, - "description": desc, - "visibility_level": visibility_level - } - r = requests.post(url, headers=self.headers, json=data) - return r.status_code == 200 or r.status_code == 201 - - - def list_repos(self) -> list: - list_groups_url = self.api + '/groups' - all_groups = [] - all_repos = [] - r = requests.get(list_groups_url, headers=self.headers) - if r.status_code == 200: - repos = r.json() - for repo in repos: - all_groups.append(repo["id"]) - for group in all_groups: - group_detail_url = list_groups_url + "/" + str(group) - r = requests.get(group_detail_url, headers=self.headers) - if r.status_code == 200: - group_detail = r.json() - for repo in group_detail["projects"]: - all_repos.append( - dict(name=repo["name_with_namespace"], - desc=repo['description'], - is_private=not repo['public']) - ) - return all_repos - - -if __name__ == "__main__": - if not os.path.isdir(settings.TEMP_DIR): - os.mkdir(settings.TEMP_DIR) - - source_type = settings.SOURCE_GIT.get('type', '') - dest_type = settings.DEST_GIT.get('type', '') - if source_type == 'gitlab': - source_git = Gitlab(settings.SOURCE_GIT) - elif source_type == 'github': - source_git = Github(settings.SOURCE_GIT) - elif source_type == 'coding': - source_git = Coding(settings.SOURCE_GIT) - elif source_type in ['gitea', 'gogs']: - source_git = Gitea(settings.SOURCE_GIT) - elif source_type == 'gitee': - source_git = Gitee(settings.SOURCE_GIT) - elif source_type == "gf": - source_git = GF(settings.SOURCE_GIT) - else: - raise ValueError(f'暂不支持此类Git服务器: {source_type}') - - if dest_type == 'gitlab': - dest_git = Gitlab(settings.DEST_GIT) - elif dest_type == 'github': - dest_git = Github(settings.DEST_GIT) - elif dest_type == 'coding': - raise ValueError(f'暂不支持迁移至 Coding') - elif dest_type in ['gitea', 'gogs']: - dest_git = Gitea(settings.DEST_GIT) - elif dest_type == 'gitee': - dest_git = Gitee(settings.DEST_GIT) - elif dest_type == "gf": - dest_type = GF(settings.DEST_GIT) - else: - raise ValueError(f'暂不支持此类Git服务器: {source_type}') - - all_repos = source_git.list_repos() - for i, repo in enumerate(all_repos): - print(f'{str(i)}. {repo["name"]}') - repo_ids = [int(i) for i in input('请输入需要迁移的仓库序号, 以英文逗号分割: ').replace(' ', '').split(',')] - for repo_id in repo_ids: - migrate_repo = all_repos[repo_id] - try: - repo_dir = source_git.clone_repo(migrate_repo['name']) - if repo_dir: - has_create = dest_git.create_repo(**migrate_repo) - if has_create: - dest_git.push_repo(migrate_repo['name'], repo_dir) - except Exception as e: - logger.error(e) - - - diff --git a/gigrator/__init__.py b/gigrator/__init__.py new file mode 100644 index 0000000..3697aee --- /dev/null +++ b/gigrator/__init__.py @@ -0,0 +1,4 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" diff --git a/gigrator/config.py b/gigrator/config.py new file mode 100644 index 0000000..82e3bb1 --- /dev/null +++ b/gigrator/config.py @@ -0,0 +1,78 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +import yaml +from gigrator.git.git import Git +from gigrator.git import gitlab, github, gitee, gitea, coding, gongfeng + +# https://pyyaml.org/wiki/PyYAMLDocumentation +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +def load_config(cfg_file: str) -> dict: + with open(cfg_file, "r") as f: + cfg = yaml.load(f, Loader=Loader) + + return cfg + + +def git_factory(cfg: dict) -> Git: + provider = cfg.get("provider", "") + if not provider: + raise RuntimeError("Invalid provider") + + if provider == "gitlab": + return gitlab.Gitlab(cfg) + if provider == "github": + return github.Github(cfg) + if provider == "coding": + return coding.Coding(cfg) + if provider in ["gitea", "gogs"]: + return gitea.Gitea(cfg) + if provider == "gitee": + return gitee.Gitee(cfg) + if provider == "gf": + return gongfeng.GF(cfg) + + raise ValueError(f"Invalid provider: {provider}") + + +def prepare_migrate(cfg: dict): + migrate_cfg = cfg.get("migrate", None) + if not migrate_cfg: + raise RuntimeError("Invalid migrate") + + # 源 Git + migrate_from = migrate_cfg.get("from", None) + if not migrate_from: + raise RuntimeError("Invalid migrate.from") + migrate_from_cfg = cfg.get(migrate_from, None) + if not migrate_from_cfg: + raise ValueError("Not found: migrate.from cfg") + from_git = git_factory(migrate_from_cfg) + + # 目标 Git + migrate_to = migrate_cfg.get("to", None) + if not migrate_to: + raise RuntimeError("Invalid migrate.to") + migrate_to_cfg = cfg.get(migrate_to, None) + if not migrate_to_cfg: + raise ValueError("Not found: migrate.to cfg") + to_git = git_factory(migrate_to_cfg) + + all_repos = from_git.list_repos() + repos = [] + cfg_repos = migrate_cfg.get("repos", []) + if len(cfg_repos) == 0: + repos = all_repos + else: + for repo in all_repos: + for repo_name in cfg_repos: + if repo["name"] == repo_name: + repos.append(repo) + + return from_git, to_git, repos diff --git a/gigrator/gigrator.py b/gigrator/gigrator.py new file mode 100644 index 0000000..22347e6 --- /dev/null +++ b/gigrator/gigrator.py @@ -0,0 +1,42 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from gigrator.util import git_version +from gigrator.config import load_config, prepare_migrate +import argparse + + +def precheck(): + git_version() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Git repositories migration tool.") + parser.add_argument("-c", "--config", dest="cfg_file", default="./config.yml", + help="config file (default: ./config.yml)") + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + cfg = load_config(args.cfg_file) + + precheck() + + from_git, to_git, repos = prepare_migrate(cfg) + for repo in repos: + try: + repo_dir = from_git.clone_repo(repo["name"]) + if repo_dir: + has_create = to_git.create_repo(**repo) + if has_create: + to_git.push_repo(repo["name"], repo_dir) + except Exception as e: + raise RuntimeError(e) + + +if __name__ == "__main__": + main() diff --git a/gigrator/git/__init__.py b/gigrator/git/__init__.py new file mode 100644 index 0000000..3697aee --- /dev/null +++ b/gigrator/git/__init__.py @@ -0,0 +1,4 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" diff --git a/gigrator/git/coding.py b/gigrator/git/coding.py new file mode 100644 index 0000000..ba4f190 --- /dev/null +++ b/gigrator/git/coding.py @@ -0,0 +1,67 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +import requests +from gigrator.git.git import Git + + +class Coding(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "Authorization": f"token {self.token}" + } + + def is_repo_existed(self, repo_name: str) -> bool: + """ + GET /api/user/{username}/project/{project_name} + """ + url = f"{self.api}/api/user/{self.username}/project/{repo_name}" + with requests.get(url, headers=self.headers) as r: + if r.status_code == requests.codes.ok: + data = r.json() + return data["code"] == 0 and data["data"]["name"] == repo_name + return False + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + raise PermissionError("Coding 不支持通过API创建仓库") + + def list_repos(self) -> list: + """ + 当前用户的项目列表 + https://open.coding.net/api-reference/%E9%A1%B9%E7%9B%AE.html#%E5%BD%93%E5%89%8D%E7%94%A8%E6%88%B7%E7%9A%84%E9%A1%B9%E7%9B%AE%E5%88%97%E8%A1%A8 + GET /api/user/projects?type=all&page=1&pageSize=10 + Response 包含 totalPage + """ + url = f"{self.api}/api/user/projects" + params = { + "type": "all", + "pageSize": 10, + "page": 1 + } + total_page = 1 + all_repos = [] + while True: + if params["page"] > total_page: + break + + with requests.get(url, headers=self.headers, params=params) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + data = r.json() + if data["code"] != 0: + raise RuntimeError(data) + + total_page = data["data"]["totalPage"] + params["page"] += 1 + + repos = data["data"]["list"] + for repo in repos: + if str.lower(repo["owner_user_name"]) == str.lower(self.username): + is_private = not repo["is_public"] + all_repos.append(dict(name=repo["name"], + desc=repo["description"], + is_private=is_private)) + return all_repos diff --git a/gigrator/git/git.py b/gigrator/git/git.py new file mode 100644 index 0000000..d23e0d3 --- /dev/null +++ b/gigrator/git/git.py @@ -0,0 +1,113 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +import os +import re +import subprocess + + +class Git: + provider = "" + username = "" + token = "" + api = "" + ssh_prefix = "" + https_prefix = "" + use_https = False + headers = {} + + def __init__(self, config: dict): + self.provider = config.get("provider", "").lower() + if self.provider == "gogs": + self.provider = "gitea" + if not self.provider: + raise ValueError("Invalid provider") + + self.username = config.get("username", "") + if not self.username: + raise ValueError("Invalid username") + + self.token = config.get("token", "") + if not self.token: + raise ValueError("Invalid token") + + self.ssh_prefix = config.get("ssh_prefix", "") + if self.ssh_prefix.endswith(":"): + self.ssh_prefix = self.ssh_prefix.rstrip(":") + + self.https_prefix = config.get("https_prefix", "") + if self.https_prefix.endswith("/"): + self.https_prefix = self.https_prefix.rstrip("/") + self.https_prefix_auth = self._https_prefix_auth() + + self.use_https = config.get("use_https", False) + + self.api = config.get("api", "") + if self.api: + if not re.match(r"^http(s)?://.+$", self.api): + raise ValueError("Invalid api") + self.api = self.api.rstrip("/") + + self.clone_dir = config.get("clone_dir", "") + if not self.clone_dir: + self.clone_dir = os.path.join(os.getcwd(), + ".gigrator", + self.provider) + else: + self.clone_dir = os.path.join(self.clone_dir, + self.provider) + os.makedirs(self.clone_dir, exist_ok=True) + + def _https_prefix_auth(self) -> str: + parts = self.https_prefix.split("://") + schema = parts[0] + domain = parts[1] + return f"{schema}://{self.username}:{self.token}@{domain}" + + def clone_repo(self, repo_name: str, repo_owner: str = "") -> str: + if not repo_owner: + repo_owner = self.username + remote_addr = f"{self.ssh_prefix}:{repo_owner}/{repo_name}.git" + + if self.use_https: + remote_addr = f"{self.https_prefix_auth}/{repo_owner}/{repo_name}.git" + + clone_cmd = ["git", "clone", "--bare", remote_addr] + ret = subprocess.run(args=clone_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + cwd=self.clone_dir) + if ret.returncode == 0: + return os.path.join(self.clone_dir, repo_name + ".git") + print(ret.stderr, end='') + return None + + def push_repo(self, repo_name: str, repo_dir: str, repo_owner: str) -> bool: + if not repo_owner: + repo_owner = self.username + remote_addr = f"{self.ssh_prefix}:{repo_owner}/{repo_name}.git" + + if self.use_https: + remote_addr = f"{self.https_prefix_auth}/{repo_owner}/{repo_name}.git" + + clone_cmd = ["git", "push", "--mirror", remote_addr] + ret = subprocess.run(args=clone_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + cwd=self.clone_dir) + if ret.returncode == 0: + return os.path.join(self.clone_dir, repo_name + ".git") + print(ret.stderr, end='') + return None + + def list_repos(self) -> list: + raise NotImplementedError + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + raise NotImplementedError + + def is_repo_existed(self, repo_name: str) -> bool: + raise NotImplementedError diff --git a/gigrator/git/gitea.py b/gigrator/git/gitea.py new file mode 100644 index 0000000..5bbc58d --- /dev/null +++ b/gigrator/git/gitea.py @@ -0,0 +1,53 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from gigrator.git.git import Git +import requests + + +class Gitea(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "Content-Type": "application/json", + "Authorization": f"token {self.token}" + } + + def is_repo_existed(self, repo_name: str) -> bool: + # GET + # ​/repos​/{owner}​/{repo} + # Get a repository + url = f"{self.api}/repos/{self.username}/{repo_name}" + with requests.get(url, headers=self.headers) as r: + return r.status_code == requests.codes.ok + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + data = { + "auto_init": False, + "description": desc, + "name": name, + "private": is_private + } + url = f"{self.api}/user/repos" + with requests.post(url, headers=self.headers, json=data) as r: + return r.status_code == requests.codes.created + + def list_repos(self) -> list: + # GET + # ​/user​/repos + # List the repos that the authenticated user owns or has access to + # 没有做分页: https://github.com/go-gitea/gitea/issues/7515 + url = f"{self.api}/user/repos" + all_repos = [] + with requests.get(url, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + repos = r.json() + for repo in repos: + if repo["owner"]["username"] == self.username: + all_repos.append(dict(name=repo["name"], + desc=repo["description"], + is_private=repo["private"])) + return all_repos diff --git a/gigrator/git/gitee.py b/gigrator/git/gitee.py new file mode 100644 index 0000000..11dc8a0 --- /dev/null +++ b/gigrator/git/gitee.py @@ -0,0 +1,67 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from gigrator.git.git import Git +import requests + + +class Gitee(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "Content-Type": "application/json;charset=UTF-8" + } + + def is_repo_existed(self, repo_name: str) -> bool: + """ + 获取用户的某个仓库: GET /repos/{owner}/{repo} + https://gitee.com/api/v5/swagger#/getV5ReposOwnerRepo + """ + url = f"{self.api}/repos/{self.username}/{repo_name}" + params = { + "access_token": self.token + } + with requests.get(url, headers=self.headers, params=params) as r: + return r.status_code == requests.codes.ok + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + data = { + "access_token": self.token, + "name": name, + "description": desc, + "private": is_private + } + url = f"{self.api}/user/repos" + with requests.post(url, json=data, headers=self.headers) as r: + return r.status_code == requests.codes.created + + def list_repos(self) -> list: + """ + 列出授权用户的所有仓库: GET /user/repos + https://gitee.com/api/v5/swagger#/getV5UserRepos + """ + url = f"{self.api}/user/repos" + params = { + "access_token": self.token, + "type": "personal", + "sort": "full_name", + "per_page": 100, + "page": 1 + } + all_repos = [] + while True: + with requests.get(url, headers=self.headers, params=params) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + repos = r.json() + if len(repos) == 0: + break + + for repo in repos: + all_repos.append(dict(name=repo["name"], + desc=repo["description"], + is_private=repo["private"])) + params["page"] += 1 + return all_repos diff --git a/gigrator/git/github.py b/gigrator/git/github.py new file mode 100644 index 0000000..e37628b --- /dev/null +++ b/gigrator/git/github.py @@ -0,0 +1,125 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from gigrator.git.git import Git +import requests + + +class Github(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "Authorization": "token " + self.token + } + + def is_repo_existed(self, repo_name: str) -> bool: + query = """ + query ($repo_owner: String!, $repo_name: String!) { + repository(owner: $repo_owner, name: $repo_name) { + id + } + } + """ + variables = { + "repo_owner": self.username, + "repo_name": repo_name + } + payload = { + "query": query, + "variables": variables + } + with requests.post(self.api, json=payload, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + data = r.json() + try: + return data["data"]["repository"].get("id", None) is not None + except KeyError: + return False + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + mutation = """ + mutation ($name: String!, $desc: String!, $isPrivate: RepositoryVisibility!) { + createRepository(input: {name: $name, description: $desc, visibility: $isPrivate}) { + clientMutationId + repository { + id + } + } + } + """ + variables = { + "name": name, + "desc": desc, + "isPrivate": "PRIVATE" if is_private else "PUBLIC" + } + payload = { + "query": mutation, + "variables": variables + } + with requests.post(self.api, json=payload, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + data = r.json() + return "errors" not in data.keys() + + def list_repos(self) -> list: + query = """ + query ($first: Int!, $after: String) { + viewer { + repositories(first: $first, after: $after, ownerAffiliations: [OWNER]) { + edges { + node { + name + isPrivate + description + } + cursor + } + pageInfo { + hasNextPage + } + } + } + } + """ + variables = { + "first": 100 + } + payload = { + "query": query, + "variables": variables + } + + all_repos = [] + def parse_data(data): + repos = data["data"]["viewer"]["repositories"]["edges"] + for repo in repos: + repo = repo["node"] + all_repos.append(dict(name=repo["name"], + desc=repo["description"], + is_private=repo["isPrivate"])) + while True: + with requests.post(self.api, json=payload, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + data = r.json() + try: + parse_data(data) + has_next_page = data["data"]["viewer"]["repositories"]["pageInfo"]["hasNextPage"] + if not has_next_page: + break + + variables["after"] = data["data"]["viewer"]["repositories"]["edges"][-1]["cursor"] + payload = { + "query": query, + "variables": variables + } + except Exception as e: + print(data) + raise RuntimeError(e) + return all_repos diff --git a/gigrator/git/gitlab.py b/gigrator/git/gitlab.py new file mode 100644 index 0000000..76b5bd1 --- /dev/null +++ b/gigrator/git/gitlab.py @@ -0,0 +1,62 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from gigrator.git.git import Git +import requests +from urllib.parse import quote + + +class Gitlab(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "Private-Token": self.token + } + + def is_repo_existed(self, repo_name: str) -> bool: + """ + Get single project + GET /projects/:id + """ + path = quote(f"{self.username}/{repo_name}", safe="") + url = f"{self.api}/projects/{path}" + with requests.get(url, headers=self.headers) as r: + return r.status_code == requests.codes.ok + + def create_repo(self, name: str, desc: str, is_private: bool) -> bool: + data = { + "name": name, + "description": desc, + "visibility": "private" if is_private else "public" + } + url = f"{self.api}/projects" + with requests.post(url, json=data, headers=self.headers) as r: + return r.status_code == requests.codes.created + + def list_repos(self) -> list: + """ + List user projects: GET /users/:user_id/projects (需要分页: ?page=1) + 不存在401的问题: 会返回公开的仓库 + """ + url = f"{self.api}/users/{self.username}/projects" + params = { + "page": 1 + } + all_repos = [] + while True: + with requests.get(url, headers=self.headers, params=params) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content) + + repos = r.json() + if len(repos) == 0: + break + + params["page"] += 1 + for repo in repos: + is_private = repo["visibility"] != "public" + all_repos.append(dict(name=repo["name"], + desc=repo["description"], + is_private=is_private)) + return all_repos diff --git a/gigrator/git/gongfeng.py b/gigrator/git/gongfeng.py new file mode 100644 index 0000000..c9d4d81 --- /dev/null +++ b/gigrator/git/gongfeng.py @@ -0,0 +1,68 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +from urllib.parse import urlencode +import requests +from gigrator.git.git import Git + + +class GF(Git): + def __init__(self, config: dict): + super().__init__(config) + self.headers = { + "PRIVATE-TOKEN": self.token + } + + def is_repo_existed(self, repo_name: str) -> bool: + # repo_name如果包含命名空间需要URL编码 + repo_name_encoded = urlencode(repo_name) + url = f"{self.api}/projects/{repo_name_encoded}/repository/tree" + with requests.get(url, headers=self.headers) as r: + return r.status_code == requests.codes.ok + + # repo_name = name_with_namespace + def create_repo(self, repo_name: str, desc: str, is_private: bool) -> bool: + url = f"{self.api}/projects" + repo_full_name_list = repo_name.split("/") + if len(repo_full_name_list) == 1: + repo_name_namespace = "" + repo_name_short = repo_name + else: + repo_name_namespace = repo_full_name_list[0] + repo_name_short = repo_full_name_list[1] + visibility_level = 0 if is_private else 10 + payload = { + "name": repo_name_short, + "namespace_id": repo_name_namespace, + "description": desc, + "visibility_level": visibility_level + } + with requests.post(url, headers=self.headers, json=payload) as r: + return r.status_code == requests.codes.ok or r.status_code == requests.codes.created + + def list_repos(self) -> list: + list_groups_url = f"{self.api}/groups" + all_repos = [] + with requests.get(list_groups_url, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + all_groups = [] + groups = r.json() + for group in groups: + all_groups.append(group["id"]) + + for group_id in all_groups: + group_detail_url = f"{list_groups_url}/{str(group_id)}" + with requests.get(group_detail_url, headers=self.headers) as r: + if r.status_code != requests.codes.ok: + raise RuntimeError(r.content.decode("utf-8")) + + group_detail = r.json() + for repo in group_detail["projects"]: + is_private = not repo["public"] + all_repos.append(dict(name=repo["name_with_namespace"], + desc=repo["description"], + is_private=is_private)) + return all_repos diff --git a/gigrator/util.py b/gigrator/util.py new file mode 100644 index 0000000..da871bc --- /dev/null +++ b/gigrator/util.py @@ -0,0 +1,18 @@ +""" +author: K8sCat +link: https://github.com/k8scat/gigrator.git +""" +import subprocess +import sys + + +def git_version(): + ret = subprocess.run(args=["git", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8") + if ret.returncode == 0: + print(ret.stdout, end='') + else: + print(ret.stderr, end='') + sys.exit(1) diff --git a/images/gigrator.png b/images/gigrator.png deleted file mode 100644 index d73eb6b..0000000 Binary files a/images/gigrator.png and /dev/null differ diff --git a/requirements.txt b/requirements.txt index 81e7055..cc1dcda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -requests==2.22.0 -urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +requests==2.26.0 +PyYAML==5.4.1 \ No newline at end of file diff --git a/settings.py b/settings.py deleted file mode 100644 index 787cb18..0000000 --- a/settings.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -@author: hsowan -@date: 2020/2/10 - -""" -import os - -# type Git服务器类型, 例如: gitee, github, gitlab, gitea, gogs, coding, 必填 -# username 所在Git服务器的用户名, 必填 -# token 用户所在Git服务器的授权令牌, 必填 -# url Git服务器的访问地址, 例如: https://git.example.com (包括具体协议: http/https) -# 需要设置url的Git服务器有: gitlab, gitea, gogs, 其他Git服务器默认为空即可 -# 源Git服务器配置 -SOURCE_GIT = { - 'type': '', - 'username': '', - 'token': '', - 'url': '' -} -# 目的Git服务器配置 -DEST_GIT = { - 'type': 'gitlab', - 'username': 'hsowan', - 'token': '8MUV39-3fHTyn5-Vf2n6', - 'url': 'https://git.ncucoder.com' -} - -# 支持的Git服务器 -SUPPORT_GITS = ['gitlab', 'github', 'gitee', 'gitea', 'coding', 'gogs', 'gf'] - -# 仓库暂存目录 -TEMP_DIR = os.path.join(os.path.dirname(__file__), '.repos') - -# GitLab -GITLAB_API_VERSION = '/api/v4' - -# GitHub -# 暂不支持GitHub Enterprise -GITHUB_API = 'https://api.github.com/graphql' -GITHUB_SSH_PREFIX = 'git@github.com:' - -# 码云 -GITEE_API = 'https://gitee.com/api/v5' -GITEE_SSH_PREFIX = 'git@gitee.com:' - -# Coding -# 暂不支持私有部署 -CODING_SSH_PREFIX = 'git@e.coding.net:' - -# Gitea/Gogs -GITEA_API_VERSION = '/api/v1' - -# 腾讯工蜂 -# 私有化部署请修改地址 -GF_SSH_PREFIX = 'git@code.tencent.com' -GF_API = 'https://code.tencent.com/api/v3' \ No newline at end of file diff --git a/settings_example.py b/settings_example.py deleted file mode 100644 index 212428d..0000000 --- a/settings_example.py +++ /dev/null @@ -1,39 +0,0 @@ -# GitHub 配置 -GITHUB = { - 'type': 'github', - 'username': 'your_username', - 'token': 'your_token', - 'url': '' # 不用填写 -} - -# Gitee -GITEE = { - 'type': 'gitee', - 'username': 'your_username', - 'token': 'your_token', - 'url': '' # 不用填写 -} - -# GitLab -GITLAB = { - 'type': 'gitlab', - 'username': 'your_username', - 'token': 'your_token', - 'url': 'https://git.your_domain.com' # Git服务器地址 -} - -# Gitea/Gogs, -GITEA_OR_GOGS = { - 'type': 'gitea', # Gitea和Gogs都填写gitea即可 - 'username': 'your_username', - 'token': 'your_token', - 'url': 'https://git.your_domain.com' # Git服务器地址 -} - -# Coding -CODING = { - 'type': 'coding', # Gitea和Gogs都填写gitea即可 - 'username': 'your_username', # username即Coding地址的二级域名, 例如我的Coding地址是https://hsowan.coding.net, hsowan就是我的username - 'token': 'your_token', - 'url': 'https://git.your_domain.com' # Git服务器地址 -} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..09194fc --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +import setuptools + +requirements = ['PyYaml', 'requests'] + +with open("README.md", 'r', encoding="utf-8") as f: + long_description = f.read() + +setuptools.setup( + name='gigrator', + version='1.0.2', + author='K8sCat', + author_email='k8scat@gmail.com', + description='Git repositories migration tool', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/k8scat/gigrator', + packages=setuptools.find_packages(), + entry_points={'console_scripts': ['gigrator = gigrator.gigrator:main']}, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=requirements, + python_requires='>=3.6', +)