Skip to content

Commit

Permalink
Update for NetBox 4.0.7; Add more logging, error checking; Fix netbox…
Browse files Browse the repository at this point in the history 
…-community#134, netbox-community#84 (update)
  • Loading branch information
@SenilePenguin
SenilePenguin committed Jul 11, 2024
1 parent dda8ed8 commit 6252682
Show file tree
Hide file tree
Showing 5 changed files with 183 additions and 95 deletions.
30 changes: 22 additions & 8 deletions nb-dt-import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import pynetbox
from glob import glob
import os
import sys
import time

import settings
from netbox_api import NetBox
Expand All @@ -15,15 +17,19 @@ def main():
args = settings.args

netbox = NetBox(settings)
settings.handle.log("-=-=-=-=- Starting operation -=-=-=-=-")
files, vendors = settings.dtl_repo.get_devices(
f'{settings.dtl_repo.repo_path}/device-types/', args.vendors)

settings.handle.log(f'{len(vendors)} Vendors Found')
device_types = settings.dtl_repo.parse_files(files, slugs=args.slugs)
settings.handle.log(f'{len(device_types)} Device-Types Found')
settings.handle.log("Creating Manufacturers")
netbox.create_manufacturers(vendors)
settings.handle.log("Creating Device Types")
netbox.create_device_types(device_types)

settings.handle.log("-=-=-=-=- Checking Modules -=-=-=-=-")
if netbox.modules:
settings.handle.log("Modules Enabled. Creating Modules...")
files, vendors = settings.dtl_repo.get_devices(
Expand All @@ -39,16 +45,24 @@ def main():
f'Script took {(datetime.now() - startTime)} to run')
settings.handle.log(f'{netbox.counter["added"]} devices created')
settings.handle.log(f'{netbox.counter["images"]} images uploaded')
settings.handle.log(
f'{netbox.counter["updated"]} interfaces/ports updated')
settings.handle.log(
f'{netbox.counter["manufacturer"]} manufacturers created')
settings.handle.log(f'{netbox.counter["updated"]} interfaces/ports updated')
settings.handle.log(f'{netbox.counter["manufacturer"]} manufacturers created')
if settings.NETBOX_FEATURES['modules']:
settings.handle.log(
f'{netbox.counter["module_added"]} modules created')
settings.handle.log(
f'{netbox.counter["module_port_added"]} module interface / ports created')
settings.handle.log(f'{netbox.counter["module_added"]} modules created')
settings.handle.log(f'{netbox.counter["module_port_added"]} module interface / ports created')

settings.handle.log(f'{netbox.counter["connection_errors"]} connection errors corrected')
settings.handle.log("-=-=-=-=- Ending operation -=-=-=-=-")
time.sleep(5)

# Uncomment the line below while troubleshooting to pause on completion
#input("Debug pausing to review output. Press RETURN to close.")

def myexcepthook(type, value, traceback, oldhook=sys.excepthook):
oldhook(type, value, traceback)
input("Uncaught exception found. Press RETURN to continue execution.")

if __name__ == "__main__":
# Uncomment the line below while troubleshooting to pause on uncaught exceptions
#sys.excepthook = myexcepthook
main()
232 changes: 147 additions & 85 deletions netbox_api.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from collections import Counter
import copy
import time
import http
import http.client
import pynetbox
import requests
import os
Expand All @@ -17,12 +21,14 @@ def __init__(self, settings):
module_added=0,
module_port_added=0,
images=0,
connection_errors=0,
)
self.url = settings.NETBOX_URL
self.token = settings.NETBOX_TOKEN
self.handle = settings.handle
self.netbox = None
self.ignore_ssl = settings.IGNORE_SSL_ERRORS
self.retry_delay = int(settings.RETRY_DELAY)
self.modules = False
self.connect_api()
self.verify_compatibility()
Expand Down Expand Up @@ -80,65 +86,91 @@ def create_manufacturers(self, vendors):
self.handle.verbose_log(f"Error during manufacturer creation. - {request_error.error}")

def create_device_types(self, device_types_to_add):
for device_type in device_types_to_add:

# Remove file base path
src_file = device_type["src"]
del device_type["src"]

# Pre-process front/rear_image flag, remove it if present
saved_images = {}
image_base = os.path.dirname(src_file).replace("device-types","elevation-images")
for i in ["front_image","rear_image"]:
if i in device_type:
if device_type[i]:
image_glob = f"{image_base}/{device_type['slug']}.{i.split('_')[0]}.*"
images = glob.glob(image_glob, recursive=False)
if images:
saved_images[i] = images[0]
else:
self.handle.log(f"Error locating image file using '{image_glob}'")
del device_type[i]
retry_amount = 2

# Treat the original data as immutable in case we encounter a connection error.
for device_type_immutable in device_types_to_add:
# In the event we hit a ConnectionReset error on this item, we want to retry it.
# If it fails twice, assume it's an issue with the device_type
retries = 0

while retries < retry_amount:
device_type = copy.deepcopy(device_type_immutable) # Can this be a copy.copy(device_type_immutable)?

try:
dt = self.device_types.existing_device_types[device_type["model"]]
self.handle.verbose_log(f'Device Type Exists: {dt.manufacturer.name} - '
+ f'{dt.model} - {dt.id}')
except KeyError:
try:
dt = self.netbox.dcim.device_types.create(device_type)
self.counter.update({'added': 1})
self.handle.verbose_log(f'Device Type Created: {dt.manufacturer.name} - '
+ f'{dt.model} - {dt.id}')
except pynetbox.RequestError as e:
self.handle.log(f'Error {e.error} creating device type:'
f' {device_type["manufacturer"]["name"]} {device_type["model"]}')
if retries == 0:
self.handle.verbose_log(f'Processing Source File: {device_type["src"]}')
else:
self.handle.verbose_log(f'(Retry {retries}/{retry_amount}) Processing Source File: {device_type["src"]}')

# Remove file base path
src_file = device_type["src"]
del device_type["src"]

# Pre-process front/rear_image flag, remove it if present
saved_images = {}
image_base = os.path.dirname(src_file).replace("device-types","elevation-images")
for i in ["front_image","rear_image"]:
if i in device_type:
if device_type[i]:
image_glob = f"{image_base}/{device_type['slug']}.{i.split('_')[0]}.*"
images = glob.glob(image_glob, recursive=False)
if images:
saved_images[i] = images[0]
else:
self.handle.log(f"Error locating image file using '{image_glob}'")
del device_type[i]

try:
dt = self.device_types.existing_device_types[device_type["model"]]
self.handle.verbose_log(f'Device Type Exists: {dt.manufacturer.name} - {dt.model} - {dt.id}')
except KeyError:
try:
dt = self.netbox.dcim.device_types.create(device_type)
self.counter.update({'added': 1})
self.handle.verbose_log(f'Device Type Created: {dt.manufacturer.name} - {dt.model} - {dt.id}')
except pynetbox.RequestError as e:
self.handle.log(f'Error {e.error} creating device type: {device_type["manufacturer"]["name"]} {device_type["model"]}')
retries += 1
continue

if "interfaces" in device_type:
self.device_types.create_interfaces(device_type["interfaces"], dt.id)
if "power-ports" in device_type:
self.device_types.create_power_ports(device_type["power-ports"], dt.id)
if "power-port" in device_type:
self.device_types.create_power_ports(device_type["power-port"], dt.id)
if "console-ports" in device_type:
self.device_types.create_console_ports(device_type["console-ports"], dt.id)
if "power-outlets" in device_type:
self.device_types.create_power_outlets(device_type["power-outlets"], dt.id)
if "console-server-ports" in device_type:
self.device_types.create_console_server_ports(device_type["console-server-ports"], dt.id)
if "rear-ports" in device_type:
self.device_types.create_rear_ports(device_type["rear-ports"], dt.id)
if "front-ports" in device_type:
self.device_types.create_front_ports(device_type["front-ports"], dt.id)
if "device-bays" in device_type:
self.device_types.create_device_bays(device_type["device-bays"], dt.id)
if self.modules and 'module-bays' in device_type:
self.device_types.create_module_bays(device_type['module-bays'], dt.id)

# Finally, update images if any
if saved_images:
self.device_types.upload_images(self.url, self.token, saved_images, dt.id)

# We successfully processed the device. Don't retry it.
retries = retry_amount
except (http.client.RemoteDisconnected, requests.exceptions.ConnectionError) as e:
retries += 1
self.counter.update({'connection_errors': 1})
self.handle.log(f'A connection error occurred (Count: {self.counter["connection_errors"]})! Waiting {self.retry_delay} seconds then retrying... Exception: {e}')

# As a connection error has just occurred, we should give the remote end a moment then reconnect.
time.sleep(self.retry_delay)
self.connect_api()
continue

if "interfaces" in device_type:
self.device_types.create_interfaces(device_type["interfaces"], dt.id)
if "power-ports" in device_type:
self.device_types.create_power_ports(device_type["power-ports"], dt.id)
if "power-port" in device_type:
self.device_types.create_power_ports(device_type["power-port"], dt.id)
if "console-ports" in device_type:
self.device_types.create_console_ports(device_type["console-ports"], dt.id)
if "power-outlets" in device_type:
self.device_types.create_power_outlets(device_type["power-outlets"], dt.id)
if "console-server-ports" in device_type:
self.device_types.create_console_server_ports(device_type["console-server-ports"], dt.id)
if "rear-ports" in device_type:
self.device_types.create_rear_ports(device_type["rear-ports"], dt.id)
if "front-ports" in device_type:
self.device_types.create_front_ports(device_type["front-ports"], dt.id)
if "device-bays" in device_type:
self.device_types.create_device_bays(device_type["device-bays"], dt.id)
if self.modules and 'module-bays' in device_type:
self.device_types.create_module_bays(device_type['module-bays'], dt.id)

# Finally, update images if any
if saved_images:
self.device_types.upload_images(self.url, self.token, saved_images, dt.id)

def create_module_types(self, module_types):
all_module_types = {}
Expand All @@ -147,37 +179,63 @@ def create_module_types(self, module_types):
all_module_types[curr_nb_mt.manufacturer.slug] = {}

all_module_types[curr_nb_mt.manufacturer.slug][curr_nb_mt.model] = curr_nb_mt

retry_amount = 2
# Treat the original data as immutable in case we encounter a connection error.
for curr_mt_immutable in module_types:
# In the event we hit a ConnectionReset error on this item, we want to retry it.
# If it fails twice, assume it's an issue with the device_type
retries = 0

while retries < retry_amount:
curr_mt = copy.deepcopy(curr_mt_immutable) # Can this be a copy.copy(curr_mt_immutable)?

for curr_mt in module_types:
try:
module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt["model"]]
self.handle.verbose_log(f'Module Type Exists: {module_type_res.manufacturer.name} - '
+ f'{module_type_res.model} - {module_type_res.id}')
except KeyError:
try:
module_type_res = self.netbox.dcim.module_types.create(curr_mt)
self.counter.update({'module_added': 1})
self.handle.verbose_log(f'Module Type Created: {module_type_res.manufacturer.name} - '
+ f'{module_type_res.model} - {module_type_res.id}')
except pynetbox.RequestError as exce:
self.handle.log(f"Error '{exce.error}' creating module type: " +
f"{curr_mt}")

if "interfaces" in curr_mt:
self.device_types.create_module_interfaces(curr_mt["interfaces"], module_type_res.id)
if "power-ports" in curr_mt:
self.device_types.create_module_power_ports(curr_mt["power-ports"], module_type_res.id)
if "console-ports" in curr_mt:
self.device_types.create_module_console_ports(curr_mt["console-ports"], module_type_res.id)
if "power-outlets" in curr_mt:
self.device_types.create_module_power_outlets(curr_mt["power-outlets"], module_type_res.id)
if "console-server-ports" in curr_mt:
self.device_types.create_module_console_server_ports(curr_mt["console-server-ports"], module_type_res.id)
if "rear-ports" in curr_mt:
self.device_types.create_module_rear_ports(curr_mt["rear-ports"], module_type_res.id)
if "front-ports" in curr_mt:
self.device_types.create_module_front_ports(curr_mt["front-ports"], module_type_res.id)
if retries == 0:
self.handle.verbose_log(f'Processing Source File: {curr_mt["src"]}')
else:
self.handle.verbose_log(f'(Retry {retries}/{retry_amount}) Processing Source File: {curr_mt["src"]}')

try:
module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt["model"]]
self.handle.verbose_log(f'Module Type Exists: {module_type_res.manufacturer.name} - {module_type_res.model} - {module_type_res.id}')
except KeyError:
try:
module_type_res = self.netbox.dcim.module_types.create(curr_mt)
self.counter.update({'module_added': 1})
self.handle.verbose_log(f'Module Type Created: {module_type_res.manufacturer.name} - {module_type_res.model} - {module_type_res.id}')
except pynetbox.RequestError as exce:
self.handle.log(f"Error '{exce.error}' creating module type: {curr_mt["manufacturer"]} {curr_mt["model"]} {curr_mt["part_number"]}")
retries += 1
continue

if "interfaces" in curr_mt:
self.device_types.create_module_interfaces(curr_mt["interfaces"], module_type_res.id)
if "power-ports" in curr_mt:
self.device_types.create_module_power_ports(curr_mt["power-ports"], module_type_res.id)
if "console-ports" in curr_mt:
self.device_types.create_module_console_ports(curr_mt["console-ports"], module_type_res.id)
if "power-outlets" in curr_mt:
self.device_types.create_module_power_outlets(curr_mt["power-outlets"], module_type_res.id)
if "console-server-ports" in curr_mt:
self.device_types.create_module_console_server_ports(curr_mt["console-server-ports"], module_type_res.id)
if "rear-ports" in curr_mt:
self.device_types.create_module_rear_ports(curr_mt["rear-ports"], module_type_res.id)
if "front-ports" in curr_mt:
self.device_types.create_module_front_ports(curr_mt["front-ports"], module_type_res.id)

# We successfully processed the device. Don't retry it.
retries = retry_amount

except (http.client.RemoteDisconnected, requests.exceptions.ConnectionError) as e:
retries += 1
self.counter.update({'connection_errors': 1})
self.handle.log(f'A connection error occurred (Count: {self.counter["connection_errors"]})! Waiting {self.retry_delay} seconds then retrying... Exception: {e}')

# As a connection error has just occurred, we should give the remote end a moment then reconnect.
time.sleep(self.retry_delay)
self.connect_api()
continue

class DeviceTypes:
def __new__(cls, *args, **kwargs):
Expand Down Expand Up @@ -480,6 +538,10 @@ def upload_images(self,baseurl,token,images,device_type):

files = { i: (os.path.basename(f), open(f,"rb") ) for i,f in images.items() }
response = requests.patch(url, headers=headers, files=files, verify=(not self.ignore_ssl))

self.handle.log( f'Images {images} updated at {url}: {response}' )

if response.status_code == 500:
raise Exception(f"Remote server failed to write images. Ensure your media directory exists and is writable! - {response}")
else:
self.handle.log( f'Images {images} updated at {url}: {response} (Code {response.status_code})' )

self.counter["images"] += len(images)
7 changes: 6 additions & 1 deletion repo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def get_modules_path(self):
return os.path.join(self.get_absolute_path(), 'module-types')

def slug_format(self, name):
return re_sub('\W+', '-', name.lower())
return re_sub(r'\W+', '-', name.lower()) # Fix #139

def pull_repo(self):
try:
Expand Down Expand Up @@ -85,12 +85,17 @@ def get_devices(self, base_path, vendors: list = None):
def parse_files(self, files: list, slugs: list = None):
deviceTypes = []
for file in files:
self.handle.verbose_log(f"Parsing file {file}")
with open(file, 'r') as stream:
try:
data = yaml.safe_load(stream)
except yaml.YAMLError as excep:
self.handle.verbose_log(excep)
continue
except UnicodeDecodeError as excep:
self.handle.verbose_log(excep)
continue

manufacturer = data['manufacturer']
data['manufacturer'] = {
'name': manufacturer, 'slug': self.slug_format(manufacturer)}
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
GitPython==3.1.32
pynetbox==7.0.1
pynetbox==7.3.4
python-dotenv==1.0.0
PyYAML==6.0.1
Loading

0 comments on commit 6252682

Please sign in to comment.