Youngj/deep data source scrapper

backend/src/index.ts

@@ -10,24 +10,33 @@ const _PASSWORD = '799Ln38Lsjqs!yg^99wfs*9ahYo6L8';
 interface DeedData {
   county: string;
   dateOfSale: Date;
-  deedBook: number;
-  deedPage: number;
+  deedBook: number | string;
+  deedPage: number | string;
   propertyAddress: string;
-  propertyValue: number;
-  propertyTaxDue: number;
+  propertyValue: number | string;
+  propertyTaxDue: number | string;
   propertyLandDistrict: string;
-  propertyLandLot: number;
+  propertyLandLot: number | string;
   buyerName: string;
   buyerAddress: string;
   sellerName: string;
   sellerAddress: string;
 }
 
+//navigates to source of deed data
 async function navigateToSource(address: string): Promise<void> {
   const browser = await puppeteer.launch();
   const page = await browser.newPage();
 
-  //goto deed data site
+  await navigateDataSource(page, address);
+
+  await scrapeData(page, address);
+
+  browser.close();
+}
+
+//navigate to page with deed data
+async function navigateDataSource(page: puppeteer.Page, address: string) {
   await page.goto(_DATASOURCEPAGELOGIN);
 
   //click login [top right]
@@ -58,16 +67,39 @@ async function navigateToSource(address: string): Promise<void> {
     page.waitForNavigation(),
   ]);
 
+  //wait for deed data selections to load
   await page.waitForSelector('#BodyContent_lvDashboard_btnViewPT61_0');
-  //click 'view pt-61 information'
-  await page.click('#BodyContent_lvDashboard_btnViewPT61_0');
 
-  scrapeData(page, address);
+  const deedIndex = await getLatestDeedIndex(page);
+  //click 'view pt-61 information' for deed data page
+  await page.click(`#BodyContent_lvDashboard_btnViewPT61_${deedIndex}`);
+}
 
-  browser.close();
+async function getLatestDeedIndex(page: puppeteer.Page) {
+  //if more than one result for deed find latest
+  //otherwise return only deed available
+  if (await page.$('#BodyContent_lvDashboard_btnViewPT61_1')) {
+    let dateArray: Date[] = [];
+
+    for (let deedCounter = 0; deedCounter < 10; deedCounter++) {
+      if (await page.$(`#BodyContent_lvDashboard_btnViewPT61_${deedCounter}`)) {
+        let element = await page.waitForSelector(
+          `#BodyContent_lvDashboard_lblDashboardSaleDate_${deedCounter}`
+        );
+        let date = await element.evaluate((e) => e.textContent);
+        dateArray.push(new Date(date));
+      }
+    }
+    const latestDate = dateArray.reduce((a, b) => {
+      return a > b ? a : b;
+    });
+    return dateArray.indexOf(latestDate);
+  }
+  return 0;
 }
 
 async function scrapeData(page: Page, address: string): Promise<DeedData> {
+  //?? is there a more elegant way to do this?
   const county = await page.waitForSelector(
     `#BodyContent_lvFinalViews_ucCombinedQuickView_0_ucPT61QuickView_0_lblCountyName_0`
   );
@@ -110,26 +142,25 @@ async function scrapeData(page: Page, address: string): Promise<DeedData> {
   const DeedData = {
     county: await county.evaluate((e) => e.textContent),
     dateOfSale: new Date(await dateOfSale.evaluate((e) => e.textContent)),
-    deedBook: parseInt(await deedBook.evaluate((e) => e.textContent)),
-    deedPage: parseInt(await deedPage.evaluate((e) => e.textContent)),
+    deedBook: parseInt(await deedBook.evaluate((e) => e.textContent)) || '',
+    deedPage: parseInt(await deedPage.evaluate((e) => e.textContent)) || '',
     propertyAddress: address,
     propertyValue: parseInt(
       await propertyValue.evaluate((e) => e.textContent.replace(/[^0-9]/g, ''))
-    ),
+    ) || '',
     propertyTaxDue: parseInt(
       await propertyTaxDue.evaluate((e) => e.textContent.replace(/[^0-9]/g, ''))
-    ),
+    ) || '',
     propertyLandDistrict: await propertyLandDistrict.evaluate(
       (e) => e.textContent
     ),
     propertyLandLot: parseInt(
       await propertyLandLot.evaluate((e) => e.textContent)
-    ),
+    ) || '',
     buyerName: await buyerName.evaluate((e) => e.textContent),
     buyerAddress: await buyerAddress.evaluate((e) => e.textContent),
     sellerName: await sellerName.evaluate((e) => e.textContent),
     sellerAddress: await sellerAddress.evaluate((e) => e.textContent),
   };
-  console.log(DeedData);
   return DeedData;
 }

backend/tsconfig.json

@@ -20,6 +20,7 @@
     /* Type Checking */
     "strict": false,                                      /* Enable all strict type-checking options. */
     "noImplicitAny": true,                               /* Enable error reporting for expressions and declarations with an implied 'any' type. */
-    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+    "skipLibCheck": true,                                 /* Skip type checking all .d.ts files. */
+    "sourceMap": true
   }
 }