base.py

import re
import shutil
import random
import requests
from subDomainsBrute import subDomainsBruteMain
from Sublist3r import Sublist3rMain
from Subfinder import subfinderMain
from OneForAll import oneforallMain
from Ksubdomain import ksubdomainMain
from Httpx import httpxMain
import config
from ServerJiang.jiangMain import SendNotice
import os
from kscan import kscanMain
from concurrent.futures import ThreadPoolExecutor,wait, ALL_COMPLETED

'''
cleanTempXrayReport()函数
    功能：删除xray临时报告目录下的全部文件
'''
def cleanTempXrayReport():
    shutil.rmtree("{}".format(config.Xray_temp_report_path))
    os.mkdir("{}".format(config.Xray_temp_report_path))
    return

'''
subScan(target) 函数
参数：
    target 待扫描的URL
    filename 扫描目标 target 的对应md5之后的十六进制
作用：
    对输入的target进行子域名的收集，并将结果存储到队列sub_queue里
输出：
    结果保存在队列sub_queue里面，传递给队列去重函数
子域名收集整合模块：
    OneForAll
    ARL
    Knock
    subDomainsBrute
    Subfinder
    Sublist3r
    ...(可根据自己需要自行添加
'''
def subScan(domain ,filename):
    '''
    调用四个子域名搜集模块，并将结果保存在 sub_queue 里面
    使用 queueDeduplication 进行子域名 -> 网址的转换 ，同时检测存活
    :param target:
    :param filename:
    :return:
    '''
    try:
        with ThreadPoolExecutor(max_workers=5) as pool:
            task1 = pool.submit(oneforallMain.OneForAllScan, domain)
            args = [domain,filename]
            #newTask = executor.submit(lambda p: doFileParse(*p), args)
            task2 = pool.submit(lambda p: subfinderMain.subfinderScan(*p),args)
            if config.if_brute:
                task3 = pool.submit(lambda p: ksubdomainMain.ksubdomainScan(*p),args)
                all_task = [task1,task2,task3]
            else:
                all_task = [task1,task2]
            wait(all_task, return_when=ALL_COMPLETED)
            results = []
            for res in all_task:
                results = results + res.result()
            print(results)
    except Exception as e:
        print(e)
        pass
    try:
        queueDeduplication(filename,list(set(results)))
    except Exception as e:
        print(str(e))
        pass



'''
queueDeduplication(filename) 队列去重函数
参数：
    filename 扫描目标 target 的对应md5之后的十六进制
作用：
    对子域名队列sub_queue里面的元素进行去重处理
输出：
    结果保存在target_queue队列里面，存储到saveSub文件夹下对应filename.txt中并且成为待扫描的目标
'''
def queueDeduplication(filename,results):
    Sub_report_path =config.Sub_report_path +filename +".txt"
    notclean_report_path = config.Temp_path + filename + "_notcleandomain.txt"
    Url_report_path =config.Url_report_path +filename +".txt"


    with open(notclean_report_path, 'a+') as f:
        for res in results:
            f.write("{}\n".format(res))
    
    ksubdomainMain.clean(filename,notclean_report_path,Sub_report_path)
    httpxMain.httpxScan(Sub_report_path,Url_report_path)
            
        
    # if if_urlscan:
    #     try:
    #         urllist = kscanMain.urlcheck(Url_report_path,nocdn_report_path)
    #     except Exception as e:
    #         print(e)
    #         pass


        
    
    print("queueDeduplication End~")
    SendNotice("子域名搜集完毕，保存文件名:{}".format(filename))
    return



'''
checkBlackList(url)
检测目标URL是否在黑名单列表中
'''
def checkBlackList(url):
    for i in config.blacklist:
        if i in url:
            return False
    return True



def main():
    a=set()
    a.add(1)
    a.add(2)
    print(list(a))
    return

if __name__ == '__main__':
    main()