-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathapi.py
190 lines (154 loc) · 6.96 KB
/
api.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
import requests
from bs4 import BeautifulSoup
import json
import time
import os
class Gpu():
def __init__(self):
self.base_url = "https://www.techpowerup.com/gpu-specs/"
self.parameters = {
"mfgr" : ["NVIDIA","AMD","INTEL"],
"lower_year" : 2020,
"upper_year" : 2020 + 1 }
self.url_list = []
def create_url_list(self):
for year in range(self.parameters["lower_year"],self.parameters["upper_year"]):
self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][0]}&released={year}&mobile=No")
self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][1]}&released={year}&mobile=No")
#self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][2]}&released={year}&mobile=No")
#self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][0]}&released={year}&mobile=Yes")
#self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][1]}&released={year}&mobile=Yes")
#self.url_list.append(f"{self.base_url}?mfgr={self.parameters['mfgr'][2]}&released={year}&mobile=Yes")
def update_url_list(self):
self.url_list.clear()
self.create_url_list()
self.print_list(self.url_list) #testing line
self.validate_url_list()
def validate_url_list(self):
for each_url in self.url_list:
if not (requests.get(each_url).ok):
print (f"Error with URL: {each_url}")
return False
else:
print(f"URL: {each_url} OK")
time.sleep(1)
return True
def print_list(self, list_object):
for item in list_object:
print (item)
return
def create_directories(self):
parent_directory = "./data/"
year_directory = []
for each_year in range (self.parameters["lower_year"], self.parameters["upper_year"]):
each_year = str(each_year)
path = os.path.join(parent_directory, each_year)
os.mkdir(path)
year_directory.append(path)
for each_manufacturer in self.parameters["mfgr"]:
each_manufacturer = str(each_manufacturer)
path = os.path.join(each_manufacturer)
def dict_format(self, part_object):
spec_list = part_object.find_all("td")
core_counts = spec_list[7].string.split("/")
part_dict = \
{"vendor" : spec_list[0]['class'][0].replace("vendor-", ""),
"model" : str(spec_list[0].find('a').string),
"url" : spec_list[0].find('a')['href'],
"chip" : spec_list[1].string,
"release_date" : str(spec_list[2].string),
"bus" : spec_list[3].string,
"memory" : spec_list[4].string,
"gpu_clock" : int((spec_list[5]).string.replace(" MHz", "")),
"memory_clock" : int((spec_list[6]).string.replace(" MHz", "")),
"shader" : int(core_counts[0]),
"tmu" : int(core_counts[1]),
"rop" : int(core_counts[2])}
ram
mobo
cpu
print (part_dict) #testing line
if (self.dict_is_valid(part_dict)):
return part_dict
else:
return None
def dict_is_valid(self, dict_object):
condition_log = []
condition_log.append("NVIDIA" in dict_object["vendor"] or "AMD" in dict_object["vendor"] or "INTEL" in dict_object["vendor"])
condition_log.append(type(dict_object["model"]) is str)
condition_log.append("gpu-specs" in dict_object["url"])
#ondition_log.append(dict_object["chip"])
condition_log.append(type(dict_object["release_date"]) is str)
condition_log.append("PCIe" in dict_object["bus"])
condition_log.append("DDR" in dict_object["memory"] or "bit" in dict_object["memory"])
condition_log.append(type(dict_object["gpu_clock"]) is int)
condition_log.append(type(dict_object["memory_clock"]) is int)
condition_log.append(type(dict_object["shader"]) is int)
condition_log.append(type(dict_object["tmu"]) is int)
condition_log.append(type(dict_object["rop"]) is int)
print (condition_log) #testing line
if (False in condition_log):
return False
else:
return True
def get_custom_boards(self, gpu_url):
new_request = requests.get(gpu_url)
html_text = BeautifulSoup(new_request.text, "html.parser")
board_list = html_text.final_all("td", class_="has-image")
print (board_list)
def gpu_fetch(self):
for url in self.url_list:
new_request = requests.get(url)
html_text = BeautifulSoup(new_request.text, "html.parser")
html_text = html_text.find("table", class_="processors")
parts_list = html_text.find_all("tr")
del parts_list[0:2]
parts_list = list(map(self.dict_format, parts_list))
def update_reference():
pass
def query_user():
user_input = str(input("Enter GPU model: \n"))
query_object = {"quicksearch": user_input}
new_query = requests.post("https://www.techpowerup.com/gpu-specs/", json = query_object)
print (new_query.text)
#Export dict object into json file in subfolder "export"
def json_export(dict_object):
print ("Creating JSON file...\n")
new_file = open('export/data.json', 'x')
json_object = json.dumps(dict_object, indent=2)
new_file.write(json_object)
new_file.close()
print ("JSON file created.")
def help(self):
command_list = "json_export : export a json file containing gpu and cpu data" \
"gpu_list : output list of all GPUs in database" \
"cpu_list : output list of all CPUs in database" \
"find : returns True of False" \
"specs : returns detailed spec dictionary object from specified gpu or cpu (1 argument)" \
"list_all : return list of GPUs or CPUs containing string argument given" \
"compare : returns performance difference between two GPUs or CPUs (2 arguments)" \
print (command_list)
class Cpu():
def __init__(self):
self.base_url = "https://www.techpowerup.com/cpu-specs/"
self.parameters = {
"mfgr" : ["AMD","INTEL"],
"mobile" : ["No","Yes"],
"lower_year" : 2010,
"upper_year" : 2023 }
self.url_list = []
def cpu_fetch(new_request):
URL = "https://www.techpowerup.com/cpu-specs/"
html_query(URL)
html_text = BeautifulSoup(new_request.text, "html.parser")
html_text = html_text.find("table", class_="processors")
parts_list = list(html_text.find_all("a"))
for index in range(len(parts_list)):
parts_list[index] = parts_list[index].string
print (parts_list) #testing line
def main():
api = Gpu()
api.update_url_list()
api.gpu_fetch()
if __name__ == '__main__':
main()