feat: new is_official tag for feeds

README.md

@@ -53,7 +53,8 @@ Contains the JSON schemas used to validate the feeds in the integration tests.
 | mdb_source_id |  Unique ID | System generated | Unique numerical identifier for the feed.      |
 | data_type     | Enum| Required| The data format that the feed uses: `gtfs`.|
 | features      | Array of Enums | Optional | An array of features which can be any of: <ul><li>`fares-v2`</li><li>`fares-v1`</li><li>`flex-v1`</li><li>`flex-v2`</li><li>`pathways`</li></ul>|  
-| status        | Enum | Optional | Describes status of the feed. Should be one of: <ul><li>`active`: Feed should be used in public trip planners.</li><li>`deprecated`: Feed is explicitly deprecated and should not be used in public trip planners.</li><li>`inactive`: Feed hasn't been recently updated and should be used at risk of providing outdated information.</li><li>`development`: Feed is being used for development purposes and should not be used in public trip planners.</li></ul>Feed is assumed to be `active` if status is not explicitly provided.|  
+| status        | Enum | Optional | Describes status of the feed. Should be one of: <ul><li>`active`: Feed should be used in public trip planners.</li><li>`deprecated`: Feed is explicitly deprecated and should not be used in public trip planners.</li><li>`inactive`: Feed hasn't been recently updated and should be used at risk of providing outdated information.</li><li>`development`: Feed is being used for development purposes and should not be used in public trip planners.</li></ul>Feed is assumed to be `active` if status is not explicitly provided.|
+| is_official        | Enum | Optional | Flag indicating if the source comes from the agency itself or not.  <ul><li>`True`: Feed comes from the agency as an authorized source.</li><li>`False`: Feed is created by researchers or partners unaffiliated with the agency or municipality.</li></ul>Feed's is_official flag is assumed to be `False` if it is not explicitly provided.|  
 |redirect| Object | Optional | When a feed is deprecated by a provider and replaced with a new URL, redirect information is provided to point to the new feed.|
 | - id      | Foreign ID of mdb_source_id | Optional | New feed that replaces the current feed that is out of date or no longer maintained by the provider. |
 | - comment | Text | Optional | comment to explain redirect if needed (e.g new aggregate feed) |
@@ -89,7 +90,8 @@ Contains the JSON schemas used to validate the feeds in the integration tests.
 | name        |  Text |Optional              | An optional description of the feed, e.g to specify if the feed is an aggregate of multiple providers
 |note|Text| Optional|A note to clarify complex use cases for consumers, for example when several static feeds are associated with a realtime feed.  |  
 | features      | Array of Enums | Optional | An array of features which can be any of: <ul><li>`occupancy`</li></ul> |  
-| status        | Enum | Optional |  Describes status of the feed. Should be one of: <ul><li>`active`: Feed should be used in public trip planners.</li><li>`deprecated`: Feed is explicitly deprecated and should not be used in public trip planners.</li><li>`inactive`: Feed hasn't been recently updated and should be used at risk of providing outdated information.</li><li>`development`: Feed is being used for development purposes and should not be used in public trip planners.</li></ul>Feed is assumed to be `active` if status is not explicitly provided.|  |  
+| status        | Enum | Optional |  Describes status of the feed. Should be one of: <ul><li>`active`: Feed should be used in public trip planners.</li><li>`deprecated`: Feed is explicitly deprecated and should not be used in public trip planners.</li><li>`inactive`: Feed hasn't been recently updated and should be used at risk of providing outdated information.</li><li>`development`: Feed is being used for development purposes and should not be used in public trip planners.</li></ul>Feed is assumed to be `active` if status is not explicitly provided.|  | 
+| is_official        | Enum | Optional | Flag indicating if the source comes from the agency itself or not.  <ul><li>`True`: Feed comes from the agency as an authorized source.</li><li>`False`: Feed is created by researchers or partners unaffiliated with the agency or municipality.</li></ul>Feed's is_official flag is assumed to be `False` if it is not explicitly provided.|   
 |redirect| Object | Optional | When a feed is deprecated by a provider and replaced with a new URL, redirect information is provided to point to the new feed.|
 | - id      | Foreign ID of mdb_source_id | Optional | New feed that replaces the current feed that is out of date or no longer maintained by the provider. |
 | - comment | String | Optional | comment to explain redirect if needed (e.g new aggregate feed) |

schemas/gtfs_realtime_source_schema.json

@@ -171,6 +171,11 @@
           }
         }
       }
+    },
+    "is_official": {
+      "type": "string",
+      "description": "True if a feed comes directly from the agency, False if a feed is created by researchers or partners unaffiliated with the agency or municipality.",
+      "enum": ["True", "False"]
     }
   },
   "required": ["mdb_source_id", "data_type", "entity_type", "provider", "urls"]

schemas/gtfs_schedule_source_schema.json

@@ -201,6 +201,11 @@
           }
         }
       }
+    },
+    "is_official": {
+      "type": "string",
+      "description": "True if a feed comes directly from the agency, False if a feed is created by researchers or partners unaffiliated with the agency or municipality.",
+      "enum": ["True","False"]
     }
   },
   "required": ["mdb_source_id", "data_type", "provider", "location", "urls"]

tools/constants.py

@@ -90,6 +90,7 @@
 REDIRECTS = "redirect"
 REDIRECT_ID = "id"
 REDIRECT_COMMENT = "comment"
+IS_OFFICIAL = "is_official"
 
 # TIME CONSTANTS
 SIX_MONTHS_IN_WEEKS = 26

tools/operations.py

@@ -21,6 +21,7 @@
     MDB_SOURCE_ID,
     FEED_CONTACT_EMAIL,
     REDIRECTS,
+    IS_OFFICIAL
 )
 from tools.representations import GtfsScheduleSourcesCatalog, GtfsRealtimeSourcesCatalog
 
@@ -42,10 +43,11 @@ def add_gtfs_realtime_source(
     api_key_parameter_name=None,
     license_url=None,
     name=None,
-    static_reference=None,
+    static_reference=None,                                                                                                                              
     note=None,
     status=None,
     features=None,
+    is_official=None,
 ):
     """
     Add a new GTFS Realtime source to the Mobility Catalogs.
@@ -66,7 +68,8 @@ def add_gtfs_realtime_source(
         note (str, optional): Additional notes regarding the source. Defaults to None.
         status (str, optional): The status of the GTFS Realtime source. Defaults to None.
         features (list, optional): A list of features of the GTFS Realtime source. Defaults to None.
-
+        is_official (str, optional): Flag indicating if the source comes from the agency itself or not. Defaults to None.
+
     Returns:
         GtfsRealtimeSourcesCatalog: The catalog with the newly added GTFS Realtime source.
     """
@@ -84,6 +87,7 @@ def add_gtfs_realtime_source(
         LICENSE: license_url,
         STATUS: status,
         FEATURES: features,
+        IS_OFFICIAL: is_official
     }
     catalog.add(**data)
     return catalog
@@ -103,6 +107,7 @@ def update_gtfs_realtime_source(
     note=None,
     status=None,
     features=None,
+    is_official = None,
 ):
     """
     Update an existing GTFS Realtime source in the Mobility Catalogs.
@@ -124,7 +129,8 @@ def update_gtfs_realtime_source(
         note (str, optional): Additional notes regarding the source. Defaults to None.
         status (str, optional): The status of the GTFS Realtime source. Defaults to None.
         features (list, optional): A list of features of the GTFS Realtime source. Defaults to None.
-
+        s_official (str, optional): Flag indicating if the source comes from the agency itself or not. Defaults to None.
+
     Returns:
         GtfsRealtimeSourcesCatalog: The catalog with the updated GTFS Realtime source.
     """
@@ -143,6 +149,7 @@ def update_gtfs_realtime_source(
         LICENSE: license_url,
         STATUS: status,
         FEATURES: features,
+        IS_OFFICIAL: is_official,
     }
     catalog.update(**data)
     return catalog
@@ -164,6 +171,7 @@ def add_gtfs_schedule_source(
     features=None,
     feed_contact_email=None,
     redirects=None,
+    is_official=None,
 ):
     """
     Add a new GTFS Schedule source to the Mobility Catalogs.
@@ -187,7 +195,8 @@ def add_gtfs_schedule_source(
         features (list, optional): A list of features of the GTFS Schedule source. Defaults to None.
         feed_contact_email (str, optional): The contact email for the feed. Defaults to None.
         redirects (list, optional): A list of redirect information for the source. Defaults to None.
-
+        is_official (str, optional): Flag indicating if the source comes from the agency itself or not. Defaults to None.
+
     Returns:
         GtfsScheduleSourcesCatalog: The catalog with the newly added GTFS Schedule source.
     """
@@ -208,6 +217,7 @@ def add_gtfs_schedule_source(
         FEATURES: features,
         FEED_CONTACT_EMAIL: feed_contact_email,
         REDIRECTS: redirects,
+        IS_OFFICIAL: is_official,
     }
     catalog.add(**data)
     return catalog
@@ -230,6 +240,7 @@ def update_gtfs_schedule_source(
     features=None,
     feed_contact_email=None,
     redirects=None,
+    is_official=None,
 ):
     """
     Update a GTFS Schedule source in the Mobility Catalogs.
@@ -254,7 +265,8 @@ def update_gtfs_schedule_source(
         features (list, optional): A list of features of the GTFS Schedule source. Defaults to None.
         feed_contact_email (str, optional): The contact email for the feed. Defaults to None.
         redirects (list, optional): A list of redirect information for the source. Defaults to None.
-
+        is_official (str, optional): Flag indicating if the source comes from the agency itself or not. Defaults to None.
+
     Returns:
         GtfsScheduleSourcesCatalog: The catalog with the updated GTFS Schedule source.
     """
@@ -276,6 +288,7 @@ def update_gtfs_schedule_source(
         FEATURES: features,
         FEED_CONTACT_EMAIL: feed_contact_email,
         REDIRECTS: redirects,
+        IS_OFFICIAL: is_official,
     }
     catalog.update(**data)
     return catalog
@@ -469,3 +482,29 @@ def get_sources_by_feature(
             globals()[f"{catalog_cls}"]().get_sources_by_feature(feature=feature)
         )
     return dict(sorted(sources.items()))
+
+def get_sources_by_is_official(
+    is_official,
+    data_type=ALL,
+):
+    """
+    Get the sources with the given is_offical flag.
+
+    This function retrieves sources from the specified data type in the Mobility Catalogs
+    that have the given is_official flag.
+
+    Args:
+        is_official (str): The feature to filter sources by.
+        data_type (str, optional): The type of data to retrieve sources for. Defaults to ALL.
+            Possible values are 'ALL', 'GTFS', 'GTFS-RT', etc.
+
+    Returns:
+        dict: A dictionary of sorted sources with the specified is_official flag from the specified catalog.
+    """
+    source_type_map = globals()[f"{data_type.upper().replace('-', '_')}_MAP"]
+    sources = {}
+    for catalog_cls in source_type_map[CATALOGS]:
+        sources.update(
+            globals()[f"{catalog_cls}"]().get_sources_by_is_official(is_official=is_official)
+        )
+    return dict(sorted(sources.items()))

tools/representations.py

@@ -57,6 +57,7 @@
     REDIRECT_ID,
     REDIRECT_COMMENT,
     REDIRECTS,
+    IS_OFFICIAL
 )
 
 PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
@@ -135,7 +136,7 @@ def aggregate(catalog_path, id_key, entity_cls):
         catalog = {}
         for path, sub_dirs, files in os.walk(catalog_path):
             for file in files:
-                with open(os.path.join(path, file)) as fp:
+                with open(os.path.join(path, file), encoding='utf-8') as fp:
                     entity_json = json.load(fp)
                     entity_id = entity_json[id_key]
                     catalog[entity_id] = entity_cls(filename=file, **entity_json)
@@ -264,6 +265,13 @@ def get_sources_by_status(self, status):
             for source_id, source in self.catalog.items()
             if source.has_status(status)
         }
+
+    def get_sources_by_is_official(self, is_official):
+        return {
+            source_id: source.as_json()
+            for source_id, source in self.catalog.items()
+            if source.has_is_official(is_official)
+        }
 
     def add(self, **kwargs):
         mdb_source_id = self.identify(self.root)
@@ -402,6 +410,7 @@ class Source(ABC):
         authentication_info_url (str, optional): URL for authentication information.
         api_key_parameter_name (str, optional): The name of the API key parameter, if applicable.
         license_url (str, optional): URL for the license information of the data.
+        is_official (str, optional): Flag indicating if the source comes from the agency itself or not.
 
     Note:
         This class is designed to be subclassed. Subclasses must implement
@@ -416,6 +425,7 @@ def __init__(self, **kwargs):
         self.filename = kwargs.pop(FILENAME)
         self.features = kwargs.pop(FEATURES, None)
         self.status = kwargs.pop(STATUS, None)
+        self.is_official = kwargs.pop(IS_OFFICIAL, None)
         urls = kwargs.get(URLS, {})
         self.direct_download_url = urls.pop(DIRECT_DOWNLOAD)
         self.authentication_type = urls.pop(AUTHENTICATION_TYPE, None)
@@ -447,6 +457,10 @@ def has_feature(self, feature):
     def has_status(self, status):
         pass
 
+    @abstractmethod
+    def has_is_official(self, is_official):
+        pass
+
     @abstractmethod
     def is_overlapping_bounding_box(
         self, minimum_latitude, maximum_latitude, minimum_longitude, maximum_longitude
@@ -558,6 +572,7 @@ def __str__(self):
             STATUS: self.status,
             FEED_CONTACT_EMAIL: self.feed_contact_email,
             REDIRECTS: self.redirects,
+            IS_OFFICIAL: self.is_official,
         }
         return json.dumps(self.schematize(**attributes), ensure_ascii=False)
 
@@ -575,6 +590,9 @@ def has_feature(self, feature):
 
     def has_status(self, status):
         return self.status == status or (self.status is None and status == ACTIVE)
+
+    def has_is_official(self, is_official):
+        return self.is_official == is_official
 
     def is_overlapping_bounding_box(
         self, minimum_latitude, maximum_latitude, minimum_longitude, maximum_longitude
@@ -655,6 +673,9 @@ def update(self, **kwargs):
         feed_contact_email = kwargs.get(FEED_CONTACT_EMAIL)
         if feed_contact_email is not None:
             self.feed_contact_email = feed_contact_email
+        is_official = kwargs.get(IS_OFFICIAL)
+        if is_official is not None:
+            self.is_official = is_official
 
         # Update the redirects
         redirects = kwargs.get(REDIRECTS)
@@ -756,6 +777,7 @@ def schematize(cls, **kwargs):
                 LICENSE: kwargs.pop(LICENSE, None),
             },
             REDIRECTS: kwargs.pop(REDIRECTS, None),
+            IS_OFFICIAL: kwargs.pop(IS_OFFICIAL, None),
         }
         if schema[NAME] is None:
             del schema[NAME]
@@ -779,6 +801,8 @@ def schematize(cls, **kwargs):
             del schema[FEED_CONTACT_EMAIL]
         if schema[REDIRECTS] is None:
             del schema[REDIRECTS]
+        if schema[IS_OFFICIAL] is None:
+            del schema[IS_OFFICIAL]
         return schema
 
 
@@ -842,6 +866,7 @@ def __str__(self):
             LICENSE: self.license_url,
             FEATURES: self.features,
             STATUS: self.status,
+            IS_OFFICIAL: self.is_official,
         }
         return json.dumps(self.schematize(**attributes), ensure_ascii=False)
 
@@ -889,6 +914,9 @@ def has_feature(self, feature):
 
     def has_status(self, status):
         return self.status == status or (self.status is None and status == ACTIVE)
+
+    def has_is_official(self, is_official):
+        return self.is_official == is_official
 
     def is_overlapping_bounding_box(
         self, minimum_latitude, maximum_latitude, minimum_longitude, maximum_longitude
@@ -948,6 +976,9 @@ def update(self, **kwargs):
         status = kwargs.get(STATUS)
         if status is not None:
             self.status = status
+        is_official = kwargs.get(IS_OFFICIAL)
+        if is_official is not None:
+            self.is_official = is_official
         return self
 
     @classmethod
@@ -1006,6 +1037,7 @@ def schematize(cls, **kwargs):
                 API_KEY_PARAMETER_NAME: kwargs.pop(API_KEY_PARAMETER_NAME, None),
                 LICENSE: kwargs.pop(LICENSE, None),
             },
+            IS_OFFICIAL: kwargs.pop(IS_OFFICIAL, None),
         }
         if schema[NAME] is None:
             del schema[NAME]
@@ -1025,4 +1057,6 @@ def schematize(cls, **kwargs):
             del schema[FEATURES]
         if schema[STATUS] is None:
             del schema[STATUS]
+        if schema[IS_OFFICIAL] is None:
+            del schema[IS_OFFICIAL]
         return schema