changed the opengraph meta data extraction to incorporate the html body.

extruct/opengraph.py

@@ -3,46 +3,46 @@
 from extruct.utils import parse_html
 
 
-_PREFIX_PATTERN = re.compile(r'\s*(\w+):\s*([^\s]+)')
+_PREFIX_PATTERN = re.compile(r"\s*(\w+):\s*([^\s]+)")
 _OG_NAMESPACES = {
-    'og': 'http://ogp.me/ns#',
-    'music': 'http://ogp.me/ns/music#',
-    'video': 'http://ogp.me/ns/video#',
-    'article': 'http://ogp.me/ns/article#',
-    'book': 'http://ogp.me/ns/book#',
-    'profile': 'http://ogp.me/ns/profile#',
+    "og": "http://ogp.me/ns#",
+    "music": "http://ogp.me/ns/music#",
+    "video": "http://ogp.me/ns/video#",
+    "article": "http://ogp.me/ns/article#",
+    "book": "http://ogp.me/ns/book#",
+    "profile": "http://ogp.me/ns/profile#",
     # non-standard but seen in the wild
-    'product': 'http://ogp.me/ns/product#',  # ~10% of product pages with OG
+    "product": "http://ogp.me/ns/product#",  # ~10% of product pages with OG
 }
 
 
 class OpenGraphExtractor(object):
     """OpenGraph extractor following extruct API."""
 
-    def extract(self, htmlstring, base_url=None, encoding='UTF-8'):
+    def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
         tree = parse_html(htmlstring, encoding=encoding)
         return list(self.extract_items(tree, base_url=base_url))
 
     def extract_items(self, document, base_url=None):
         # OpenGraph defines a web page as a single rich object.
-        for head in document.xpath('//head'):
-            html_elems = document.head.xpath("parent::html")
-            namespaces = self.get_namespaces(
-                html_elems[0]) if html_elems else {}
-            namespaces.update(self.get_namespaces(head))
-            props = []
-            for el in head.xpath('meta[@property and @content]'):
-                prop = el.attrib['property']
-                val = el.attrib['content']
-                ns = prop.partition(':')[0]
+
+        html_elems = document.head.xpath("parent::html")
+        namespaces = self.get_namespaces(html_elems[0]) if html_elems else {}
+        props = []
+        for meta in document.xpath("//meta"):
+            namespaces.update(self.get_namespaces(meta))
+            # print(namespaces)
+            if "property" in meta.attrib and "content" in meta.attrib:
+                prop = meta.attrib["property"]
+                val = meta.attrib["content"]
+                ns = prop.partition(":")[0]
+
                 if ns in _OG_NAMESPACES:
                     namespaces[ns] = _OG_NAMESPACES[ns]
                 if ns in namespaces:
                     props.append((prop, val))
-            if props:
-                yield {'namespace': namespaces, 'properties': props}
+        if props:
+            yield {"namespace": namespaces, "properties": props}
 
     def get_namespaces(self, element):
-        return dict(
-            _PREFIX_PATTERN.findall(element.attrib.get('prefix', ''))
-        )
+        return dict(_PREFIX_PATTERN.findall(element.attrib.get("prefix", "")))

extruct/utils.py

@@ -5,12 +5,12 @@
 
 
 def parse_html(html, encoding):
-    """ Parse HTML using lxml.html.HTMLParser, return a tree """
+    """Parse HTML using lxml.html.HTMLParser, return a tree"""
     parser = lxml.html.HTMLParser(encoding=encoding)
     return lxml.html.fromstring(html, parser=parser)
 
 
 def parse_xmldom_html(html, encoding):
-    """ Parse HTML using XmlDomHTMLParser, return a tree """
+    """Parse HTML using XmlDomHTMLParser, return a tree"""
     parser = XmlDomHTMLParser(encoding=encoding)
     return lxml.html.fromstring(html, parser=parser)

tests/samples/misc/opengraph_test_2.json

@@ -0,0 +1 @@
+[{"namespace": {"og": "http://ogp.me/ns#"}, "properties": [["og:title", "freeCodeCamp.org"], ["og:site_name", "YouTube"], ["og:url", "https://www.youtube.com/channel/UC8butISFwT-Wl7EV0hUK0BQ"], ["og:image", "https://yt3.ggpht.com/ytc/AKedOLRR2uNiXJiFH-XRmtGgkdICxTuDJxCPJidKFRNCNg=s900-c-k-c0x00ffffff-no-rj"], ["og:image:width", "900"], ["og:image:height", "900"], ["og:description", "Learn to code for free."], ["og:type", "profile"], ["og:video:tag", "coding bootcamp"], ["og:video:tag", "learn to code"], ["og:video:tag", "software engineer"], ["og:video:tag", "nonprofits"], ["og:video:tag", "full stack"], ["og:video:tag", "front end"], ["og:video:tag", "developer"], ["og:video:tag", "programmer"], ["og:video:tag", "javascript"], ["og:video:tag", "python"], ["og:video:tag", "web development"], ["og:video:tag", "technology"], ["og:video:tag", "math"], ["og:video:tag", "coding"], ["og:video:tag", "css"], ["og:video:tag", "html"], ["og:video:tag", "web design"], ["og:video:tag", "data science"], ["og:video:tag", "mcahine learning"], ["og:video:tag", "linux"]]}]

tests/test_opengraph.py

@@ -11,15 +11,17 @@ class TestOpengraph(unittest.TestCase):
     maxDiff = None
 
     def _test_opengraph(self, name):
-        body = get_testdata('misc', name + '.html')
-        expected = json.loads(get_testdata('misc', name + '.json').decode('UTF-8'))
-
+        body = get_testdata("misc", name + ".html")
+        expected = json.loads(get_testdata("misc", name + ".json").decode("UTF-8"))
         opengraphe = OpenGraphExtractor()
         data = opengraphe.extract(body)
         self.assertEqual(jsonize_dict(data), expected)
 
     def test_opengraph(self):
-        self._test_opengraph('opengraph_test')
+        self._test_opengraph("opengraph_test")
+
+    def test_opengraph_2(self):
+        self._test_opengraph("opengraph_test_2")
 
     def test_opengraph_ns_product(self):
-        self._test_opengraph('opengraph_ns_product_test')
+        self._test_opengraph("opengraph_ns_product_test")