MMRotate-1.x regression benchmark

.dev_scripts/batch_test_list.py

@@ -0,0 +1,13 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+
+# Note: mAP is on DOTA trainval set here.
+
+# yapf: disable
+
+cfa = dict(
+    config='configs/cfa/cfa-qbox_r50_fpn_1x_dota.py',
+    checkpoint='atss_r50_fpn_1x_coco_20200209-985f7bd0.pth',
+    url='https://download.openmmlab.com/mmrotate/v0.1.0/cfa/cfa_r50_fpn_1x_dota_le135/cfa_r50_fpn_1x_dota_le135-aed1cbc6.pth', # noqa
+    metric=dict(mAP=69.63),
+)
+# yapf: enable

.dev_scripts/batch_train_list.txt

@@ -0,0 +1,41 @@
+configs/cfa/cfa-qbox_r50_fpn_1x_dota.py
+configs/cfa/cfa-qbox_r50_fpn_40e_dota.py
+configs/convnext/rotated-retinanet-rbox-le90_convnext-tiny_fpn_kld-stable_adamw-1x_dota.py
+configs/csl/rotated-retinanet-rbox-le90_r50_fpn_csl-gaussian_amp-1x_dota.py
+configs/gliding_vertex/gliding-vertex-rbox_r50_fpn_1x_dota.py
+configs/gwd/rotated-retinanet-hbox-oc_r50_fpn_gwd_1x_dota.py
+configs/kfiou/rotated-retinanet-hbox-le90_r50_fpn_kfiou_1x_dota.py
+configs/kfiou/rotated-retinanet-hbox-oc_r50_fpn_kfiou_1x_dota.py
+configs/kfiou/rotated-retinanet-hbox-le135_r50_fpn_kfiou_1x_dota.py
+configs/kfiou/r3det-oc_r50_fpn_kfiou-ln_1x_dota.py
+configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld_1x_dota.py
+configs/kld/rotated-retinanet-hbox-oc_r50_fpn_kld-stable_1x_dota.py
+configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_1x_dota.py
+configs/kld/r3det-oc_r50_fpn_kld_1x_dota.py
+configs/kld/r3det-oc_r50_fpn_kld-stable_1x_dota.py
+configs/kld/r3det-tiny-oc_r50_fpn_kld_1x_dota.py
+configs/kld/rotated-retinanet-rbox-le90_r50_fpn_kld-stable_adamw-1x_dota.py
+configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_amp-1x_dota.py
+configs/oriented_rcnn/oriented-rcnn-le90_r50_fpn_1x_dota.py
+configs/oriented_reppoints/oriented-reppoints-qbox_r50_fpn_1x_dota.py
+configs/r3det/r3det-oc_r50_fpn_1x_dota.py
+configs/r3det/r3det-tiny-oc_r50_fpn_1x_dota.py
+configs/redet/redet-le90_re50_refpn_amp-1x_dota.py
+configs/redet/redet-le90_re50_refpn_1x_dota.py
+configs/roi_trans/roi-trans-le90_r50_fpn_amp-1x_dota.py
+configs/roi_trans/roi-trans-le90_r50_fpn_1x_dota.py
+configs/roi_trans/roi-trans-le90_swin-tiny_fpn_1x_dota.py
+configs/rotated_atss/rotated-atss-le90_r50_fpn_1x_dota.py
+configs/rotated_faster_rcnn/rotated-faster-rcnn-le90_r50_fpn_1x_dota.py
+configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_1x_dota.py
+configs/rotated_fcos/rotated-fcos-le90_r50_fpn_1x_dota.py
+configs/rotated_fcos/rotated-fcos-hbox-le90_r50_fpn_csl-gaussian_1x_dota.py
+configs/rotated_fcos/rotated-fcos-le90_r50_fpn_kld_1x_dota.py
+configs/rotated_reppoints/rotated-reppoints-qbox_r50_fpn_1x_dota.py
+configs/rotated_retinanet/rotated-retinanet-hbox-oc_r50_fpn_1x_dota.py
+configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_1x_dota.py
+configs/rotated_retinanet/rotated-retinanet-rbox-le90_r50_fpn_amp-1x_dota.py
+configs/rotated_retinanet/rotated-retinanet-rbox-le135_r50_fpn_1x_dota.py
+configs/s2anet/s2anet-le135_r50_fpn_1x_dota.py
+configs/s2anet/s2anet-le135_r50_fpn_amp-1x_dota.py
+configs/sasm_reppoints/sasm-reppoints-qbox_r50_fpn_1x_dota.py

.dev_scripts/check_links.py

@@ -0,0 +1,157 @@
+# Modified from:
+# https://github.com/allenai/allennlp/blob/main/scripts/check_links.py
+
+import argparse
+import logging
+import os
+import pathlib
+import re
+import sys
+from multiprocessing.dummy import Pool
+from typing import NamedTuple, Optional, Tuple
+
+import requests
+from mmengine.logging import MMLogger
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Goes through all the inline-links '
+        'in markdown files and reports the breakages')
+    parser.add_argument(
+        '--num-threads',
+        type=int,
+        default=100,
+        help='Number of processes to confirm the link')
+    parser.add_argument('--https-proxy', type=str, help='https proxy')
+    parser.add_argument(
+        '--out',
+        type=str,
+        default='link_reports.txt',
+        help='output path of reports')
+    args = parser.parse_args()
+    return args
+
+
+OK_STATUS_CODES = (
+    200,
+    401,  # the resource exists but may require some sort of login.
+    403,  # ^ same
+    405,  # HEAD method not allowed.
+    # the resource exists, but our default 'Accept-' header may not
+    # match what the server can provide.
+    406,
+)
+
+
+class MatchTuple(NamedTuple):
+    source: str
+    name: str
+    link: str
+
+
+def check_link(
+        match_tuple: MatchTuple,
+        http_session: requests.Session,
+        logger: logging = None) -> Tuple[MatchTuple, bool, Optional[str]]:
+    reason: Optional[str] = None
+    if match_tuple.link.startswith('http'):
+        result_ok, reason = check_url(match_tuple, http_session)
+    else:
+        result_ok = check_path(match_tuple)
+    if logger is None:
+        print(f"  {'✓' if result_ok else '✗'} {match_tuple.link}")
+    else:
+        logger.info(f"  {'✓' if result_ok else '✗'} {match_tuple.link}")
+    return match_tuple, result_ok, reason
+
+
+def check_url(match_tuple: MatchTuple,
+              http_session: requests.Session) -> Tuple[bool, str]:
+    """Check if a URL is reachable."""
+    try:
+        result = http_session.head(
+            match_tuple.link, timeout=5, allow_redirects=True)
+        return (
+            result.ok or result.status_code in OK_STATUS_CODES,
+            f'status code = {result.status_code}',
+        )
+    except (requests.ConnectionError, requests.Timeout):
+        return False, 'connection error'
+
+
+def check_path(match_tuple: MatchTuple) -> bool:
+    """Check if a file in this repository exists."""
+    relative_path = match_tuple.link.split('#')[0]
+    full_path = os.path.join(
+        os.path.dirname(str(match_tuple.source)), relative_path)
+    return os.path.exists(full_path)
+
+
+def main():
+    args = parse_args()
+
+    # setup logger
+    logger = MMLogger.get_instance(name='mmdet', log_file=args.out)
+
+    # setup https_proxy
+    if args.https_proxy:
+        os.environ['https_proxy'] = args.https_proxy
+
+    # setup http_session
+    http_session = requests.Session()
+    for resource_prefix in ('http://', 'https://'):
+        http_session.mount(
+            resource_prefix,
+            requests.adapters.HTTPAdapter(
+                max_retries=5,
+                pool_connections=20,
+                pool_maxsize=args.num_threads),
+        )
+
+    logger.info('Finding all markdown files in the current directory...')
+
+    project_root = (pathlib.Path(__file__).parent / '..').resolve()
+    markdown_files = project_root.glob('**/*.md')
+
+    all_matches = set()
+    url_regex = re.compile(r'\[([^!][^\]]+)\]\(([^)(]+)\)')
+    for markdown_file in markdown_files:
+        with open(markdown_file) as handle:
+            for line in handle.readlines():
+                matches = url_regex.findall(line)
+                for name, link in matches:
+                    if 'localhost' not in link:
+                        all_matches.add(
+                            MatchTuple(
+                                source=str(markdown_file),
+                                name=name,
+                                link=link))
+
+    logger.info(f'  {len(all_matches)} markdown files found')
+    logger.info('Checking to make sure we can retrieve each link...')
+
+    with Pool(processes=args.num_threads) as pool:
+        results = pool.starmap(check_link, [(match, http_session, logger)
+                                            for match in list(all_matches)])
+
+    # collect unreachable results
+    unreachable_results = [(match_tuple, reason)
+                           for match_tuple, success, reason in results
+                           if not success]
+
+    if unreachable_results:
+        logger.info('================================================')
+        logger.info(f'Unreachable links ({len(unreachable_results)}):')
+        for match_tuple, reason in unreachable_results:
+            logger.info('  > Source: ' + match_tuple.source)
+            logger.info('    Name: ' + match_tuple.name)
+            logger.info('    Link: ' + match_tuple.link)
+            if reason is not None:
+                logger.info('    Reason: ' + reason)
+        sys.exit(1)
+    logger.info('No Unreachable link found.')
+
+
+if __name__ == '__main__':
+    main()

.dev_scripts/convert_test_benchmark_script.py

@@ -0,0 +1,87 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import os
+import os.path as osp
+
+from mmengine import Config
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Convert benchmark model list to script')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument('--port', type=int, default=29666, help='dist port')
+    parser.add_argument(
+        '--run', action='store_true', help='run script directly')
+    parser.add_argument(
+        '--out', type=str, help='path to save model benchmark script')
+
+    args = parser.parse_args()
+    return args
+
+
+def process_model_info(model_info):
+    config = model_info['config'].strip()
+    checkpoint = model_info['checkpoint'].strip()
+    return dict(config=config, checkpoint=checkpoint)
+
+
+def create_test_bash_info(commands, model_test_dict, port, script_name):
+    config = model_test_dict['config']
+    checkpoint = model_test_dict['checkpoint']
+
+    echo_info = f' \necho \'{config}\' &'
+    commands.append(echo_info)
+    commands.append('\n')
+
+    command_info = f'python {script_name} '
+
+    command_info += f'{config} '
+    command_info += f'$CHECKPOINT_DIR/{checkpoint} '
+
+    command_info += f'--cfg-option env_cfg.dist_cfg.port={port} '
+    command_info += ' &'
+
+    commands.append(command_info)
+
+
+def main():
+    args = parse_args()
+    if args.out:
+        out_suffix = args.out.split('.')[-1]
+        assert args.out.endswith('.sh'), \
+            f'Expected out file path suffix is .sh, but get .{out_suffix}'
+    assert args.out or args.run, \
+        ('Please specify at least one operation (save/run/ the '
+         'script) with the argument "--out" or "--run"')
+
+    commands = []
+
+    checkpoint_root = 'CHECKPOINT_DIR=$1 '
+    commands.append(checkpoint_root)
+    commands.append('\n')
+
+    script_name = osp.join('tools', 'test.py')
+    port = args.port
+
+    cfg = Config.fromfile(args.config)
+    for model_key in cfg:
+        model_infos = cfg[model_key]
+        if not isinstance(model_infos, list):
+            model_infos = [model_infos]
+        for model_info in model_infos:
+            print('processing: ', model_info['config'])
+            model_test_dict = process_model_info(model_info)
+            create_test_bash_info(commands, model_test_dict, port, script_name)
+            port += 1
+
+    command_str = ''.join(commands)
+    if args.out:
+        with open(args.out, 'w') as f:
+            f.write(command_str)
+    if args.run:
+        os.system(command_str)
+
+
+if __name__ == '__main__':
+    main()

.dev_scripts/convert_train_benchmark_script.py

@@ -0,0 +1,68 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import argparse
+import os
+import os.path as osp
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Convert benchmark model json to script')
+    parser.add_argument(
+        'txt_path', type=str, help='txt path output by benchmark_filter')
+    parser.add_argument(
+        '--run', action='store_true', help='run script directly')
+    parser.add_argument(
+        '--out', type=str, help='path to save model benchmark script')
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    if args.out:
+        out_suffix = args.out.split('.')[-1]
+        assert args.out.endswith('.sh'), \
+            f'Expected out file path suffix is .sh, but get .{out_suffix}'
+    assert args.out or args.run, \
+        ('Please specify at least one operation (save/run/ the '
+         'script) with the argument "--out" or "--run"')
+
+    root_name = './tools'
+    train_script_name = osp.join(root_name, 'train.py')
+
+    commands = []
+
+    with open(args.txt_path, 'r') as f:
+        model_cfgs = f.readlines()
+        for i, cfg in enumerate(model_cfgs):
+            cfg = cfg.strip()
+            if len(cfg) == 0:
+                continue
+            # print cfg name
+            echo_info = f'echo \'{cfg}\' &'
+            commands.append(echo_info)
+            commands.append('\n')
+
+            command_info = f'python {train_script_name} '
+            command_info += f'{cfg} '
+
+            command_info += '--cfg-options default_hooks.checkpoint.' \
+                            'max_keep_ckpts=1 '
+            command_info += '&'
+
+            commands.append(command_info)
+
+            if i < len(model_cfgs):
+                commands.append('\n')
+
+        command_str = ''.join(commands)
+        if args.out:
+            with open(args.out, 'w') as f:
+                f.write(command_str)
+        if args.run:
+            os.system(command_str)
+
+
+if __name__ == '__main__':
+    main()