Indexing EnergyData Catalog¶

Example notebook on how to index energydata ino

In [1]:

import json
from datetime import datetime
from pathlib import Path

import contextily as cx
import pandas as pd
import requests
from bs4 import BeautifulSoup
from h3ronpy.arrow import cells_parse
from h3ronpy.pandas.vector import cells_dataframe_to_geodataframe

from worldex.datasets.energydata import EnergyDataDataset

import json from datetime import datetime from pathlib import Path import contextily as cx import pandas as pd import requests from bs4 import BeautifulSoup from h3ronpy.arrow import cells_parse from h3ronpy.pandas.vector import cells_dataframe_to_geodataframe from worldex.datasets.energydata import EnergyDataDataset

In [2]:

# fetch data
url = 'https://energydata.info/api/3/action/package_search?rows=10000'
response = requests.get(url)
datasets = response.json()['result']['results']

# fetch data url = 'https://energydata.info/api/3/action/package_search?rows=10000' response = requests.get(url) datasets = response.json()['result']['results']

In [3]:

datasets_w_geo = []
for d in datasets:
    if any(r['format'] in ['GeoJSON', 'SHP ZIP', 'GPKG', 'SHP'] for r in d['resources']):
        datasets_w_geo.append(d)

datasets_w_geo = [] for d in datasets: if any(r['format'] in ['GeoJSON', 'SHP ZIP', 'GPKG', 'SHP'] for r in d['resources']): datasets_w_geo.append(d)

In [10]:

data = datasets_w_geo[1]
data

data = datasets_w_geo[1] data

Out[10]:

{'author': 'Fang Fang, Daron Bedrosyan, Clara Ivanescu',
 'author_email': 'fang.fang@neo.nl, dbedrosyan@worldbank.org, civanescu@worldbank.org',
 'country_code': ['ETH'],
 'creator_user_id': 'ac5e2088-0f44-420b-ba13-73c045dc4033',
 'end_date': '',
 'group': 'Technical Rooftop Solar Potential Assessment',
 'id': '4ab91bb0-9e03-4d18-978d-8e4d110271e8',
 'isopen': True,
 'license_id': 'CC-BY-4.0',
 'license_title': 'Creative Commons Attribution 4.0',
 'license_url': 'https://creativecommons.org/licenses/by/4.0/',
 'metadata_created': '2020-11-24T13:54:33.892745',
 'metadata_modified': '2022-04-15T15:36:18.343471',
 'name': 'addis-abeba-rooftop-solar-potential-mapping',
 'notes': 'This dataset contains solar rooftop potential data (suitable rooftop area, installable capacity, estimated yearly electricity generation, and building type ) at individual building structure level for a sample area of interest in Addis Ababa. The data was gathered by extracting building rooftop footprint polygons from very high-resolution satellite stereo imagery of 0.5m resolution. The rooftop angle, obstruction, and shading were taken into account during suitable area calculation. The results can be aggregated by different sectors or administrative units for further analysis, which is useful for planning and decision making. Detailed methodology is available upon specific request.',
 'num_resources': 3,
 'num_tags': 4,
 'organization': {'id': '6c36a5eb-4cd6-4616-b530-da46f8b04ed3',
  'name': 'neo',
  'title': 'NEO',
  'type': 'organization',
  'description': "NEO (https://www.neo.nl/home-int/) is a leading company in the Earth Observation (EO) business based in the Netherlands.  Established in 1996, NEO has 25 years track record in providing information services starting from the changes in our habitat. From the changes we detect, we built value-added services supporting both public and private organizations.\r\n\r\nNEO's EO-based solutions cover a wide range of applications including but not limited to: Energy and Asset management, Infrastructure monitoring, Environmental monitoring, Agriculture monitoring, as well as Land and water. \r\n\r\nWith the strong expertise in geo-information/remote sensing and the patented and certified solution (ISO9001 and ISO27001) SignalEyes® for monitoring with earth observation data, NEO translates geodata into value-added services for the clients.\r\n\r\nNEO serves over 250 customers in The Netherlands, within the European Union and internationally. Its services empower our customers to make better decisions, reduce costs, increase safety and prevent errors, to comply with regulations and to manage our environment better.",
  'image_url': '2020-11-19-173002.651278logo-NEO-English.jpg',
  'created': '2020-11-19T17:30:02.703348',
  'is_organization': True,
  'approval_status': 'approved',
  'state': 'active'},
 'owner_org': '6c36a5eb-4cd6-4616-b530-da46f8b04ed3',
 'private': False,
 'ref_system': 'WGS/UTM37N',
 'region': ['AFR'],
 'release_date': '2020',
 'start_date': '',
 'state': 'active',
 'status': 'Complete',
 'title': 'Addis Ababa - Rooftop Solar Potential Mapping',
 'topic': ['Energy efficiency', 'Renewable energy'],
 'type': 'dataset',
 'url': '',
 'version': '',
 'resources': [{'cache_last_updated': None,
   'cache_url': None,
   'created': '2020-11-24T14:01:27.250430',
   'datastore_active': False,
   'description': 'This vector dataset contains building footprint polygons for the area of study in the center areas of Addis Abeba. The metrics such as suitable area, installable capacity and estimated yearly potential are derived at individual building unit level. Another useful information attached is the building type.',
   'format': 'SHP ZIP',
   'hash': '',
   'id': '80b6ca4f-b41b-4c94-9c7e-1dfd00451515',
   'last_modified': '2020-11-24T14:01:27.083814',
   'metadata_modified': '2020-11-24T14:01:27.250430',
   'mimetype': 'application/zip',
   'mimetype_inner': None,
   'name': 'AddisAbeba_rooftop_potential.zip',
   'package_id': '4ab91bb0-9e03-4d18-978d-8e4d110271e8',
   'position': 0,
   'resource_type': None,
   'size': 60761869,
   'state': 'active',
   'tilejson': False,
   'url': 'https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/80b6ca4f-b41b-4c94-9c7e-1dfd00451515/download/addisabeba_rooftop_potential.zip',
   'url_type': 'upload'},
  {'cache_last_updated': None,
   'cache_url': None,
   'created': '2020-11-24T14:02:22.504421',
   'datastore_active': False,
   'description': '',
   'format': 'XLS',
   'hash': '',
   'id': '6b8f02d3-5b34-497d-9874-b252bb1440de',
   'last_modified': '2020-11-24T14:02:22.347117',
   'metadata_modified': '2020-11-24T14:02:22.504421',
   'mimetype': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
   'mimetype_inner': None,
   'name': 'AddisAbeba_result_metadata.xlsx',
   'package_id': '4ab91bb0-9e03-4d18-978d-8e4d110271e8',
   'position': 1,
   'resource_type': None,
   'size': 10423,
   'state': 'active',
   'tilejson': False,
   'url': 'https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/6b8f02d3-5b34-497d-9874-b252bb1440de/download/addisabeba_result_metadata.xlsx',
   'url_type': 'upload'},
  {'cache_last_updated': None,
   'cache_url': None,
   'created': '2022-04-15T15:36:18.367222',
   'datastore_active': False,
   'description': 'The dataset with attributes is provided also in csv format. The available attribute information are : total surface area(m2), suitable area(m2), peak_installable_capacity (kw), energy potential(Kwh), estimated building height (m), estimated roof angle (degree) and assumed building type.\r\n\r\nThis dataset covers the AddisAbaba city center area.',
   'format': 'CSV',
   'hash': '',
   'id': '2a3572b9-4d01-47d7-8238-dd47cd9ef549',
   'last_modified': '2022-04-15T15:36:18.175388',
   'metadata_modified': '2022-04-15T15:36:18.367222',
   'mimetype': 'text/csv',
   'mimetype_inner': None,
   'name': 'AddisAbaba_rooftop_solarpotential_v1.csv',
   'package_id': '4ab91bb0-9e03-4d18-978d-8e4d110271e8',
   'position': 2,
   'resource_type': None,
   'size': 16136,
   'state': 'active',
   'tilejson': False,
   'url': 'https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/2a3572b9-4d01-47d7-8238-dd47cd9ef549/download/addisababa_rooftop_solarpotential_v1.csv',
   'url_type': 'upload'}],
 'groups': [],
 'relationships_as_subject': [],
 'relationships_as_object': []}

In [11]:

path = Path(f"../data/energydata/{data['name']}")
path.mkdir(parents=True, exist_ok=True)

path = Path(f"../data/energydata/{data['name']}") path.mkdir(parents=True, exist_ok=True)

In [12]:

energyinfo_dataset = EnergyDataDataset.from_dataset(data)
energyinfo_dataset.set_dir(path)
energyinfo_dataset.index()

energyinfo_dataset = EnergyDataDataset.from_dataset(data) energyinfo_dataset.set_dir(path) energyinfo_dataset.index()

Out[12]:

	h3_index
0	88529b4de7fffff
1	88529b4d81fffff
2	88529b4d03fffff
3	88529b4e87fffff
4	88529b4ea5fffff
...	...
416	88529b79e9fffff
417	88529b4e91fffff
418	887acb4a39fffff
419	88529b7b2bfffff
420	887acb48b1fffff

421 rows × 1 columns

In [13]:

with open(path / "metadata.json") as f:
    metadata = json.load(f)

metadata

with open(path / "metadata.json") as f: metadata = json.load(f) metadata

Out[13]:

{'id': 'b1a49d30-950d-4ae8-a471-5f11e4ce6a32',
 'name': 'Addis Ababa - Rooftop Solar Potential Mapping',
 'source_org': 'EnergyInfo',
 'last_fetched': '2024-03-22T10:05:11.838877',
 'files': ['https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/80b6ca4f-b41b-4c94-9c7e-1dfd00451515/download/addisabeba_rooftop_potential.zip',
  'https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/6b8f02d3-5b34-497d-9874-b252bb1440de/download/addisabeba_result_metadata.xlsx',
  'https://energydata.info/dataset/4ab91bb0-9e03-4d18-978d-8e4d110271e8/resource/2a3572b9-4d01-47d7-8238-dd47cd9ef549/download/addisababa_rooftop_solarpotential_v1.csv'],
 'description': 'This dataset contains solar rooftop potential data (suitable rooftop area, installable capacity, estimated yearly electricity generation, and building type ) at individual building structure level for a sample area of interest in Addis Ababa. The data was gathered by extracting building rooftop footprint polygons from very high-resolution satellite stereo imagery of 0.5m resolution. The rooftop angle, obstruction, and shading were taken into account during suitable area calculation. The results can be aggregated by different sectors or administrative units for further analysis, which is useful for planning and decision making. Detailed methodology is available upon specific request.',
 'data_format': '',
 'projection': '',
 'properties': {'id': 'addis-abeba-rooftop-solar-potential-mapping'},
 'bbox': 'POLYGON ((38.8682847773977542 8.8561617321855621, 38.8682847773977542 9.0712453476984383, 38.6648141554202738 9.0712453476984383, 38.6648141554202738 8.8561617321855621, 38.8682847773977542 8.8561617321855621))',
 'keywords': [],
 'date_start': None,
 'date_end': None,
 'accessibility': 'public/open',
 'url': 'https://energydata.info/dataset/addis-abeba-rooftop-solar-potential-mapping'}

In [14]:

h3 = pd.read_parquet(path / "h3-compact.parquet")

h3_gdf = cells_dataframe_to_geodataframe(
    pd.DataFrame({"cell": cells_parse(h3.h3_index)})
)
h3_gdf_reprojected = h3_gdf.to_crs(epsg=3857)
ax = h3_gdf_reprojected.plot(figsize=(10, 10), alpha=0.5, edgecolor="k")
cx.add_basemap(ax, source=cx.providers.CartoDB.Positron)

h3 = pd.read_parquet(path / "h3-compact.parquet") h3_gdf = cells_dataframe_to_geodataframe( pd.DataFrame({"cell": cells_parse(h3.h3_index)}) ) h3_gdf_reprojected = h3_gdf.to_crs(epsg=3857) ax = h3_gdf_reprojected.plot(figsize=(10, 10), alpha=0.5, edgecolor="k") cx.add_basemap(ax, source=cx.providers.CartoDB.Positron)

In [ ]: